diff --git a/API-INTERNAL.md b/API-INTERNAL.md index 901f6791a..df46837f7 100644 --- a/API-INTERNAL.md +++ b/API-INTERNAL.md @@ -90,16 +90,16 @@ The connection layer (createStore) owns connection/transport recovery; this oper capacity recovery (eviction) so that a given failure is retried by exactly one layer:

@@ -321,16 +321,16 @@ Handles storage operation failures based on the error class (see lib/storage/err The connection layer (createStore) owns connection/transport recovery; this operation layer owns capacity recovery (eviction) so that a given failure is retried by exactly one layer: - INVALID_DATA: logs an alert and throws (the same data will always fail). -- TRANSIENT / FATAL: the connection layer already retried (transient) or exhausted its heal budget - and alerted (fatal). Retrying here would only re-amplify, so we skip the write quietly. +- TRANSIENT / FATAL: the connection layer already retried (transient) or spent a heal attempt + (fatal). Retrying here would only re-amplify, so we skip the write quietly. - CAPACITY: evicts the least recently accessed evictable key and retries, under a session-level circuit breaker (see lib/StorageCircuitBreaker.ts) that halts the loop once eviction stops making progress or failures storm — the per-operation budget alone cannot stop a session-wide storm. - DISK_PRESSURE: the device disk itself is full (or the database files are unreadable), so neither retries nor in-DB eviction can free space — the write is dropped (cache stays authoritative) with a single throttled alert + quota snapshot per burst. -- UNAVAILABLE: the storage engine does not exist in this environment, so the storage layer has - already degraded to the in-memory provider. No retry. +- UNAVAILABLE: the storage engine does not exist in this environment or its heal budget ran out, so + the storage layer has already degraded to the in-memory provider. No retry. - UNKNOWN: the provider couldn't classify it — log the full error shape (name + message + provider) once so it's visible, then bounded retry without eviction. diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index c7ddb5478..4e5cd19cf 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -567,16 +567,16 @@ function reportStorageQuota(error?: Error): Promise { * The connection layer (createStore) owns connection/transport recovery; this operation layer owns * capacity recovery (eviction) so that a given failure is retried by exactly one layer: * - INVALID_DATA: logs an alert and throws (the same data will always fail). - * - TRANSIENT / FATAL: the connection layer already retried (transient) or exhausted its heal budget - * and alerted (fatal). Retrying here would only re-amplify, so we skip the write quietly. + * - TRANSIENT / FATAL: the connection layer already retried (transient) or spent a heal attempt + * (fatal). Retrying here would only re-amplify, so we skip the write quietly. * - CAPACITY: evicts the least recently accessed evictable key and retries, under a session-level * circuit breaker (see lib/StorageCircuitBreaker.ts) that halts the loop once eviction stops making * progress or failures storm — the per-operation budget alone cannot stop a session-wide storm. * - DISK_PRESSURE: the device disk itself is full (or the database files are unreadable), so neither * retries nor in-DB eviction can free space — the write is dropped (cache stays authoritative) with * a single throttled alert + quota snapshot per burst. - * - UNAVAILABLE: the storage engine does not exist in this environment, so the storage layer has - * already degraded to the in-memory provider. No retry. + * - UNAVAILABLE: the storage engine does not exist in this environment or its heal budget ran out, so + * the storage layer has already degraded to the in-memory provider. No retry. * - UNKNOWN: the provider couldn't classify it — log the full error shape (name + message + * provider) once so it's visible, then bounded retry without eviction. */ @@ -920,6 +920,10 @@ function initializeWithDefaultKeyStates(): Promise { .catch((error) => { Logger.logAlert(`Failed to load data from storage during init. The app will boot with default key states only. Error: ${error}`); + if (Storage.classifyError(error) === StorageErrorClass.FATAL) { + Storage.degradeToMemoryOnly(error); + } + // Populate the key index so getAllKeys() returns correct results for default keys. // Without this, subscribers that check getAllKeys() would see an empty set even // though we have default values in cache. diff --git a/lib/storage/__mocks__/index.ts b/lib/storage/__mocks__/index.ts index 90c75fd62..273c1cd2b 100644 --- a/lib/storage/__mocks__/index.ts +++ b/lib/storage/__mocks__/index.ts @@ -19,6 +19,7 @@ const StorageMock = { init, classifyError: jest.fn(classifyError), getStorageProvider: jest.fn(() => MemoryOnlyProvider), + degradeToMemoryOnly: jest.fn(), getItem: jest.fn(MemoryOnlyProvider.getItem), multiGet: jest.fn(MemoryOnlyProvider.multiGet), setItem: jest.fn(MemoryOnlyProvider.setItem), diff --git a/lib/storage/errors.ts b/lib/storage/errors.ts index 3e9bba7af..f47bddff8 100644 --- a/lib/storage/errors.ts +++ b/lib/storage/errors.ts @@ -22,12 +22,12 @@ const StorageErrorClass = { DISK_PRESSURE: 'diskPressure', /** Non-serializable payload. Never retriable — the same data will always fail. */ INVALID_DATA: 'invalidData', - /** Backing-store corruption. Owner: connection layer — budgeted heal, then give up. */ + /** Backing-store corruption. Owner: connection layer — budgeted heal, then rethrow as UNAVAILABLE. */ FATAL: 'fatal', /** The storage engine itself does not exist in this environment (for example `indexedDB` is an - * undeclared global in Chrome for iOS private tabs and in Lockdown Mode). Never retriable — the - * engine cannot appear mid-session. Owner: the storage layer — degrade - * to the in-memory provider. */ + * undeclared global in Chrome for iOS private tabs and in Lockdown Mode), or a FATAL heal budget + * ran out. Never retriable — the engine cannot recover mid-session. Owner: the storage layer — + * degrade to the in-memory provider. */ UNAVAILABLE: 'unavailable', /** Unmatched by the active provider. Owner: operation layer — bounded retry, and log the shape so * recurring cases can be promoted into one of the classes above. */ diff --git a/lib/storage/index.ts b/lib/storage/index.ts index 13f245b40..0c641e7da 100644 --- a/lib/storage/index.ts +++ b/lib/storage/index.ts @@ -22,6 +22,7 @@ const initPromise = new Promise((resolve) => { type Storage = { getStorageProvider: () => StorageProvider; + degradeToMemoryOnly: (error: Error) => void; } & Omit, 'name' | 'store'>; /** @@ -70,6 +71,8 @@ const storage: Storage = { return provider; }, + degradeToMemoryOnly: (error) => degradePerformance(error), + /** * Classifies a write error using the platform provider's own classifier. Synchronous and pure — * never wrapped in tryOrDegradePerformance. diff --git a/lib/storage/providers/IDBKeyValProvider/classifyError.ts b/lib/storage/providers/IDBKeyValProvider/classifyError.ts index af94e9cb1..f7c5f2934 100644 --- a/lib/storage/providers/IDBKeyValProvider/classifyError.ts +++ b/lib/storage/providers/IDBKeyValProvider/classifyError.ts @@ -1,6 +1,6 @@ import type {ValueOf} from 'type-fest'; import {StorageErrorClass, getErrorParts} from '../../errors'; -import {INDEXED_DB_UNAVAILABLE_MESSAGE} from './isIndexedDBAvailable'; +import IDBErrorMessage from './errorMessages'; /** * Classifies an IndexedDB write failure into the shared storage taxonomy (lib/storage/errors.ts). @@ -11,7 +11,12 @@ function classifyIDBError(error: unknown): ValueOf { const {name, message} = getErrorParts(error); // The engine is absent, not broken e.g. private tabs and Lockdown Mode on WebKit. - if (message.includes("can't find variable: indexeddb") || message.includes('indexeddb is not defined') || message.includes(INDEXED_DB_UNAVAILABLE_MESSAGE.toLowerCase())) { + if (message.includes("can't find variable: indexeddb") || message.includes('indexeddb is not defined') || message.includes(IDBErrorMessage.UNAVAILABLE.toLowerCase())) { + return StorageErrorClass.UNAVAILABLE; + } + + // Reopen budget spent on FATAL errors, so the store is treated as gone for the rest of the session. + if (message.includes(IDBErrorMessage.HEAL_EXHAUSTED.toLowerCase())) { return StorageErrorClass.UNAVAILABLE; } diff --git a/lib/storage/providers/IDBKeyValProvider/createStore.ts b/lib/storage/providers/IDBKeyValProvider/createStore.ts index 5315f9c98..eb776b20d 100644 --- a/lib/storage/providers/IDBKeyValProvider/createStore.ts +++ b/lib/storage/providers/IDBKeyValProvider/createStore.ts @@ -3,7 +3,8 @@ import type {UseStore} from 'idb-keyval'; import * as Logger from '../../../Logger'; import {StorageErrorClass} from '../../errors'; import classifyIDBError from './classifyError'; -import isIndexedDBAvailable, {INDEXED_DB_UNAVAILABLE_MESSAGE} from './isIndexedDBAvailable'; +import IDBErrorMessage from './errorMessages'; +import isIndexedDBAvailable from './isIndexedDBAvailable'; const HEAL_ATTEMPTS_MAX = 3; @@ -59,7 +60,7 @@ function createStore(dbName: string, storeName: string): UseStore { if (dbp) return dbp; if (!isIndexedDBAvailable()) { - return Promise.reject(new Error(INDEXED_DB_UNAVAILABLE_MESSAGE)); + return Promise.reject(new Error(IDBErrorMessage.UNAVAILABLE)); } const request = indexedDB.open(dbName); @@ -204,11 +205,16 @@ function createStore(dbName: string, storeName: string): UseStore { } if (errorClass === StorageErrorClass.FATAL) { + const errorMessage = error instanceof Error ? error.message : String(error); Logger.logAlert('IDB heal: backing store error — heal budget exhausted, giving up', { dbName, storeName, + errorMessage, }); - } else if (errorClass === StorageErrorClass.UNKNOWN) { + throw new Error(`${IDBErrorMessage.HEAL_EXHAUSTED}: ${errorMessage}`, {cause: error}); + } + + if (errorClass === StorageErrorClass.UNKNOWN) { // UNKNOWN — unexpected at this layer; record it so it's visible. CAPACITY is the // expected propagation path (the operation layer owns its logging, and suppresses it // entirely once the circuit breaker is open), so we do NOT log it here — doing so was a diff --git a/lib/storage/providers/IDBKeyValProvider/errorMessages.ts b/lib/storage/providers/IDBKeyValProvider/errorMessages.ts new file mode 100644 index 000000000..eda4e5018 --- /dev/null +++ b/lib/storage/providers/IDBKeyValProvider/errorMessages.ts @@ -0,0 +1,7 @@ +/** Messages of errors thrown by the IndexedDB provider itself. `classifyIDBError` maps both to `StorageErrorClass.UNAVAILABLE`. */ +const IDBErrorMessage = { + UNAVAILABLE: 'IndexedDB is not available in this environment', + HEAL_EXHAUSTED: 'IndexedDB heal budget exhausted', +} as const; + +export default IDBErrorMessage; diff --git a/lib/storage/providers/IDBKeyValProvider/index.ts b/lib/storage/providers/IDBKeyValProvider/index.ts index 3870321a2..546c9acd2 100644 --- a/lib/storage/providers/IDBKeyValProvider/index.ts +++ b/lib/storage/providers/IDBKeyValProvider/index.ts @@ -5,7 +5,8 @@ import type StorageProvider from '../types'; import type {OnyxKey, OnyxValue} from '../../../types'; import createStore from './createStore'; import classifyIDBError from './classifyError'; -import isIndexedDBAvailable, {INDEXED_DB_UNAVAILABLE_MESSAGE} from './isIndexedDBAvailable'; +import IDBErrorMessage from './errorMessages'; +import isIndexedDBAvailable from './isIndexedDBAvailable'; import type {StorageKeyValuePair} from '../types'; const DB_NAME = 'OnyxDB'; @@ -40,7 +41,7 @@ const provider: StorageProvider = { */ init() { if (!isIndexedDBAvailable()) { - throw new Error(`IDBKeyVal store could not be created: ${INDEXED_DB_UNAVAILABLE_MESSAGE}`); + throw new Error(`IDBKeyVal store could not be created: ${IDBErrorMessage.UNAVAILABLE}`); } const newIdbKeyValStore = createStore(DB_NAME, STORE_NAME); diff --git a/lib/storage/providers/IDBKeyValProvider/isIndexedDBAvailable.ts b/lib/storage/providers/IDBKeyValProvider/isIndexedDBAvailable.ts index cabd8c30b..8e660421e 100644 --- a/lib/storage/providers/IDBKeyValProvider/isIndexedDBAvailable.ts +++ b/lib/storage/providers/IDBKeyValProvider/isIndexedDBAvailable.ts @@ -1,6 +1,3 @@ -/** Matched by `classifyIDBError` so a guarded failure classifies as UNAVAILABLE like an unguarded one. */ -const INDEXED_DB_UNAVAILABLE_MESSAGE = 'indexedDB is not available in this environment'; - /** * `typeof` instead of `indexedDB === undefined`: on WebKit the global is undeclared in private tabs and * Lockdown Mode, so reading it throws a `ReferenceError` instead of evaluating to `undefined`. @@ -10,4 +7,3 @@ function isIndexedDBAvailable(): boolean { } export default isIndexedDBAvailable; -export {INDEXED_DB_UNAVAILABLE_MESSAGE}; diff --git a/tests/unit/onyxUtilsTest.ts b/tests/unit/onyxUtilsTest.ts index 779786958..8efcb575b 100644 --- a/tests/unit/onyxUtilsTest.ts +++ b/tests/unit/onyxUtilsTest.ts @@ -1555,6 +1555,25 @@ describe('OnyxUtils', () => { }); }); + describe('initializeWithDefaultKeyStates', () => { + it('should degrade to memory-only when the init read fails with a FATAL error', async () => { + const internalError = Object.assign(new Error('Internal error.'), {name: 'UnknownError'}); + jest.mocked(StorageMock.getAll).mockRejectedValueOnce(internalError); + + await OnyxUtils.initializeWithDefaultKeyStates(); + + expect(StorageMock.degradeToMemoryOnly).toHaveBeenCalledWith(internalError); + }); + + it('should not degrade to memory-only when the init read fails with an UNKNOWN error', async () => { + jest.mocked(StorageMock.getAll).mockRejectedValueOnce(new Error('some brand new failure')); + + await OnyxUtils.initializeWithDefaultKeyStates(); + + expect(StorageMock.degradeToMemoryOnly).not.toHaveBeenCalled(); + }); + }); + describe('afterInit', () => { beforeEach(() => { // Resets the deferred init task before each test. diff --git a/tests/unit/storage/providers/classifyErrorTest.ts b/tests/unit/storage/providers/classifyErrorTest.ts index a0051b570..6e6b01aa8 100644 --- a/tests/unit/storage/providers/classifyErrorTest.ts +++ b/tests/unit/storage/providers/classifyErrorTest.ts @@ -21,6 +21,8 @@ describe('classifyIDBError', () => { [new ReferenceError("Can't find variable: indexedDB"), StorageErrorClass.UNAVAILABLE], [new ReferenceError('indexedDB is not defined'), StorageErrorClass.UNAVAILABLE], [new Error('indexedDB is not available in this environment'), StorageErrorClass.UNAVAILABLE], + // FATAL heal budget spent. + [new Error('IndexedDB heal budget exhausted: Internal error.'), StorageErrorClass.UNAVAILABLE], // Anything else stays UNKNOWN. [new Error('some brand new failure'), StorageErrorClass.UNKNOWN], ])('classifies %s as %s', (error, expectedClass) => { diff --git a/tests/unit/storage/providers/createStoreTest.ts b/tests/unit/storage/providers/createStoreTest.ts index 9f2d8214f..a5d43d40a 100644 --- a/tests/unit/storage/providers/createStoreTest.ts +++ b/tests/unit/storage/providers/createStoreTest.ts @@ -3,6 +3,7 @@ import createStore from '../../../../lib/storage/providers/IDBKeyValProvider/cre import * as Logger from '../../../../lib/Logger'; import {StorageErrorClass} from '../../../../lib/storage/errors'; import classifyIDBError from '../../../../lib/storage/providers/IDBKeyValProvider/classifyError'; +import IDBErrorMessage from '../../../../lib/storage/providers/IDBKeyValProvider/errorMessages'; const STORE_NAME = 'teststore'; let testDbCounter = 0; @@ -60,7 +61,7 @@ describe('createStore', () => { await withoutIndexedDB(async () => { const operation = store('readonly', (s) => IDB.promisifyRequest(s.get('key1'))); - await expect(operation).rejects.toThrow('indexedDB is not available in this environment'); + await expect(operation).rejects.toThrow(IDBErrorMessage.UNAVAILABLE); await expect(operation.catch((error: unknown) => classifyIDBError(error))).resolves.toBe(StorageErrorClass.UNAVAILABLE); }); }); @@ -69,7 +70,7 @@ describe('createStore', () => { const store = createStore(uniqueDBName(), STORE_NAME); await withoutIndexedDB(async () => { - await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('key1')))).rejects.toThrow('indexedDB is not available in this environment'); + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('key1')))).rejects.toThrow(IDBErrorMessage.UNAVAILABLE); }); expect(logInfoSpy).not.toHaveBeenCalledWith(expect.stringContaining('IDB transient error'), expect.anything()); @@ -81,7 +82,7 @@ describe('createStore', () => { const store = createStore(uniqueDBName(), STORE_NAME); await withoutIndexedDB(async () => { - await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('key1')))).rejects.toThrow('indexedDB is not available in this environment'); + await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('key1')))).rejects.toThrow(IDBErrorMessage.UNAVAILABLE); }); await store('readwrite', (s) => { @@ -471,7 +472,11 @@ describe('createStore', () => { // Budget exhausted — 4th call should NOT attempt healing, but should log budget exhausted logAlertSpy.mockClear(); - await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow('Internal error opening backing store'); + const exhaustedError = await store('readonly', (s) => IDB.promisifyRequest(s.get('k'))).catch((error: unknown) => error); + expect(exhaustedError).toBeInstanceOf(Error); + expect((exhaustedError as Error).message).toBe('IndexedDB heal budget exhausted: Internal error opening backing store for indexedDB.open.'); + expect((exhaustedError as Error).cause).toBeInstanceOf(DOMException); + expect(classifyIDBError(exhaustedError)).toBe(StorageErrorClass.UNAVAILABLE); expect(logAlertSpy).toHaveBeenCalledWith(expect.stringContaining('heal budget exhausted'), expect.anything()); expect(logAlertSpy).not.toHaveBeenCalledWith(expect.stringContaining('dropping cached connection and reopening'), expect.anything()); }); diff --git a/tests/unit/storage/tryOrDegradePerformanceTest.ts b/tests/unit/storage/tryOrDegradePerformanceTest.ts index b5b2a4162..cf6177e26 100644 --- a/tests/unit/storage/tryOrDegradePerformanceTest.ts +++ b/tests/unit/storage/tryOrDegradePerformanceTest.ts @@ -105,6 +105,29 @@ describe('storage/tryOrDegradePerformance', () => { expect(storage.getStorageProvider().name).toBe('MemoryOnlyProvider'); }); + it('should fall back to MemoryOnlyProvider once the IndexedDB heal budget is exhausted', async () => { + const {storage} = loadIsolatedStorage(); + + storage.init(); + + const targetError = new Error('IndexedDB heal budget exhausted: Internal error.', {cause: new DOMException('Internal error.', 'UnknownError')}); + storage.getStorageProvider().setItem = jest.fn().mockReturnValue(Promise.reject(targetError)); + + await expect(storage.setItem('key', {test: 'data'})).rejects.toBe(targetError); + + expect(storage.getStorageProvider().name).toBe('MemoryOnlyProvider'); + await storage.setItem('key', {test: 'data'}); + await expect(storage.getItem('key')).resolves.toEqual({test: 'data'}); + }); + + it('should fall back to MemoryOnlyProvider when degradeToMemoryOnly is called', () => { + const {storage} = loadIsolatedStorage(); + + storage.degradeToMemoryOnly(new DOMException('Internal error.', 'UnknownError')); + + expect(storage.getStorageProvider().name).toBe('MemoryOnlyProvider'); + }); + it('should still classify the error as UNAVAILABLE after degrading to MemoryOnlyProvider', async () => { const {storage} = loadIsolatedStorage();