From 706905557c19c9b8a958fd6f4abf28ddd26db436 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Wed, 4 Nov 2026 12:49:09 +0700 Subject: [PATCH 1/4] feat(extension,web): extension observation v1, stock and sku, flagged ingest - shared-data: ExtensionObservation v1 in rules, plain functions with no zod. The parser names the first broken field the way the DOS-Me RPC does in DETAIL. No ratings, all keys required, listing host must match the marketplace, and observed_at must be within the last 24 hours (up to 10 minutes ahead). - Extension: the extractor reads stock state and quantity (JSON-LD availability and inventoryLevel, product meta) and the SKU (Lazada and Tiki URLs, JSON-LD sku). buildExtensionObservationV1 maps a signal to v1. Nothing new is sent: the lookup and the opt-in payload are unchanged, and a test pins that. - Web: POST and DELETE /api/v1/rate/extension/submitter and POST /api/v1/rate/extension/observation call the narrow M2 RPCs as anon. They answer 404 NOT_ENABLED unless RATE_EXTENSION_OBSERVATIONS=1, and observations also need RATE_NETWORK_HASH_SALT. Only an HMAC of the /24 or /48 reaches the database, never the raw IP. Refs #1606, #1175 Co-Authored-By: Claude Opus 5.5 --- .../src/extractors/ecommerce-detector.ts | 148 +++++++++ apps/extension/src/security-guards.ts | 63 ++++ apps/extension/src/test-extension-pipeline.ts | 150 +++++++++ apps/extension/src/types.ts | 12 + .../api/v1/rate/__tests__/write-paths.test.ts | 230 ++++++++++++++ .../v1/rate/extension/observation/route.ts | 109 +++++++ .../api/v1/rate/extension/submitter/route.ts | 113 +++++++ .../src/server/rate/extension-observations.ts | 174 +++++++++++ .../src/__tests__/shared-data.test.ts | 105 +++++++ .../src/rules/extension-observation.ts | 290 ++++++++++++++++++ packages/shared-data/src/rules/index.ts | 1 + 11 files changed, 1395 insertions(+) create mode 100644 apps/web/src/app/api/v1/rate/extension/observation/route.ts create mode 100644 apps/web/src/app/api/v1/rate/extension/submitter/route.ts create mode 100644 apps/web/src/server/rate/extension-observations.ts create mode 100644 packages/shared-data/src/rules/extension-observation.ts diff --git a/apps/extension/src/extractors/ecommerce-detector.ts b/apps/extension/src/extractors/ecommerce-detector.ts index 46c9e6221..79803c325 100644 --- a/apps/extension/src/extractors/ecommerce-detector.ts +++ b/apps/extension/src/extractors/ecommerce-detector.ts @@ -1,5 +1,6 @@ import type { ExtractedPageProductSignal, + ExtractedStockState, SupportedEcommercePlatform, } from '../types.js'; @@ -473,6 +474,115 @@ export function extractPriceFromJsonLdOffers(offersRaw: unknown): { return { price: resolvedPrice, originalPrice: validOriginal }; } +/** + * A schema.org availability (`https://schema.org/InStock`, `InStock`, or a + * page's own "in stock" text) as a stock state; null when it says nothing + * (DOS/Rate#1606). Pre-orders and back-orders are unknown, not in stock. + */ +export function stockStateFromAvailability( + raw: unknown +): ExtractedStockState | null { + if (typeof raw !== 'string' || raw.length > 200) return null; + const v = raw + .toLowerCase() + .replace(/^https?:\/\/schema\.org\//, '') + .replace(/[\s_-]+/g, ''); + if (!v) return null; + if (/^(?:instock|instoreonly|onlineonly|available)$/.test(v)) { + return 'in_stock'; + } + if (/^(?:limitedavailability|lowstock)$/.test(v)) return 'low_stock'; + if (/^(?:outofstock|soldout|discontinued|oos)$/.test(v)) { + return 'out_of_stock'; + } + if (/^(?:preorder|presale|backorder|madetoorder)$/.test(v)) { + return 'unknown'; + } + return null; +} + +function stockQtyOf(raw: unknown): number | null { + const value = + raw && typeof raw === 'object' && !Array.isArray(raw) + ? (raw as Record).value + : raw; + const n = + typeof value === 'number' + ? value + : typeof value === 'string' && /^\d{1,7}$/.test(value.trim()) + ? Number(value.trim()) + : Number.NaN; + return Number.isInteger(n) && n >= 0 && n <= 1_000_000 ? n : null; +} + +/** Stock state and quantity from JSON-LD offers: the first offer that says. */ +export function extractStockFromJsonLdOffers(offersRaw: unknown): { + stockState: ExtractedStockState | null; + stockQty: number | null; +} { + const offers: Record[] = []; + const push = (entry: unknown) => { + if ( + entry && + typeof entry === 'object' && + !Array.isArray(entry) && + offers.length < MAX_JSON_LD_OFFERS + ) { + offers.push(entry as Record); + } + }; + for (const entry of Array.isArray(offersRaw) ? offersRaw : [offersRaw]) { + push(entry); + const nested = (entry as { offers?: unknown } | null)?.offers; + if (Array.isArray(nested)) nested.forEach(push); + } + for (const offer of offers) { + const stockState = stockStateFromAvailability(offer.availability); + if (stockState) { + return { + stockState, + stockQty: + stockState === 'out_of_stock' ? 0 : stockQtyOf(offer.inventoryLevel), + }; + } + } + return { stockState: null, stockQty: null }; +} + +const SKU_ID_RE = /^[A-Za-z0-9_.-]{1,64}$/; + +/** + * The id of the selected variant: from the URL where the store puts it there + * (Lazada `-s.html`, Tiki `?spid=`), else the JSON-LD `sku` (DOS/Rate#1606). + * `rawUrl` is the page URL before cleaning, since Tiki keeps it in the query. + */ +export function extractSkuId( + platform: SupportedEcommercePlatform, + rawUrl: string, + jsonLdSku: unknown +): string | null { + try { + const url = new URL(rawUrl); + if (platform === 'lazada') { + const m = url.pathname.match(/-s(\d{1,20})\.html$/i); + if (m?.[1]) return m[1]; + } + if (platform === 'tiki') { + const spid = url.searchParams.get('spid'); + if (spid && /^\d{1,20}$/.test(spid)) return spid; + } + } catch { + // Fall through to the structured data. + } + const sku = + typeof jsonLdSku === 'number' && Number.isInteger(jsonLdSku) + ? String(jsonLdSku) + : typeof jsonLdSku === 'string' + ? jsonLdSku.trim() + : ''; + return SKU_ID_RE.test(sku) ? sku : null; +} + export function extractRatingFromJsonLd(aggRaw: unknown): { ratingStar: number | null; ratingCount: number | null; @@ -640,6 +750,11 @@ export function extractProductSignalFromDom( let originalPrice: number | null = null; let ratingStar: number | null = null; let ratingCount: number | null = null; + let stockState: ExtractedStockState | null = null; + let stockQty: number | null = null; + let jsonLdSku: unknown = null; + let titleFromJsonLd = false; + let priceFromJsonLd = false; // 1. Try JSON-LD Product structured data first (cleanest & resilient across SPA updates) const ldScripts = doc.querySelectorAll('script[type="application/ld+json"]'); @@ -651,11 +766,24 @@ export function extractProductSignalFromDom( if (isJsonLdProductEntity(item)) { if (!title && typeof item.name === 'string') { title = cleanExtractedProductTitle(item.name); + titleFromJsonLd = Boolean(title); + } + if (!stockState) { + const stock = extractStockFromJsonLdOffers(item.offers); + stockState = stock.stockState; + stockQty = stock.stockQty; + } + if (jsonLdSku === null) { + const offerSku = Array.isArray(item.offers) + ? (item.offers[0] as { sku?: unknown } | undefined)?.sku + : (item.offers as { sku?: unknown } | undefined)?.sku; + jsonLdSku = item.sku ?? offerSku ?? null; } if (!price || !originalPrice) { const offerPrices = extractPriceFromJsonLdOffers(item.offers); if (!price && offerPrices.price) { price = offerPrices.price; + priceFromJsonLd = true; } if ( !originalPrice && @@ -732,6 +860,18 @@ export function extractProductSignalFromDom( const selectedVariant = extractSelectedVariantFromDom(doc); + // 4. Stock from the page's product meta when JSON-LD said nothing. + if (!stockState) { + const meta = doc.querySelector( + 'meta[property="product:availability"], meta[property="og:availability"], [itemprop="availability"]' + ); + stockState = stockStateFromAvailability( + meta?.getAttribute('content') ?? meta?.getAttribute('href') + ); + if (stockState === 'out_of_stock') stockQty = 0; + } + + const fromJsonLd = [titleFromJsonLd, priceFromJsonLd || price === null]; return { platform, canonicalUrl, @@ -743,5 +883,13 @@ export function extractProductSignalFromDom( originalPrice, ratingStar, ratingCount, + skuId: extractSkuId(platform, currentUrl, jsonLdSku), + stockState: stockState ?? 'unknown', + stockQty: stockState ? stockQty : null, + extractorSource: fromJsonLd.every(Boolean) + ? 'json_ld' + : fromJsonLd.some(Boolean) + ? 'mixed' + : 'dom', }; } diff --git a/apps/extension/src/security-guards.ts b/apps/extension/src/security-guards.ts index 869032254..e5ebdc02a 100644 --- a/apps/extension/src/security-guards.ts +++ b/apps/extension/src/security-guards.ts @@ -1,3 +1,7 @@ +import { + parseExtensionObservationV1, + type ExtensionObservationV1, +} from '@repo/shared-data/rules'; import type { ExtractedPageProductSignal, SupportedEcommercePlatform, @@ -286,9 +290,68 @@ export function sanitizeExtensionProductSignal( originalPrice: validOriginalPrice, ratingStar: validRatingStar, ratingCount: validRatingCount, + // The v1 observation fields (DOS/Rate#1606), checked like the rest. + skuId: + typeof safeSignal.skuId === 'string' && + /^[A-Za-z0-9_.-]{1,64}$/.test(safeSignal.skuId) + ? safeSignal.skuId + : null, + stockState: ( + ['in_stock', 'low_stock', 'out_of_stock', 'unknown'] as const + ).includes(safeSignal.stockState as never) + ? safeSignal.stockState + : 'unknown', + stockQty: + typeof safeSignal.stockQty === 'number' && + Number.isInteger(safeSignal.stockQty) && + safeSignal.stockQty >= 0 && + safeSignal.stockQty <= 1_000_000 + ? safeSignal.stockQty + : null, + extractorSource: (['json_ld', 'dom', 'mixed'] as const).includes( + safeSignal.extractorSource as never + ) + ? safeSignal.extractorSource + : 'dom', }; } +/** + * The ExtensionObservation v1 an opted-in install would send for `signal` + * (DOS/Rate#1606), or null when the page lacks what v1 needs (an item id, a + * price). Not sent yet: the opt-in signal still uses buildCrowdIngestPayload + * until the quarantine table exists and the disclosures name v1. + */ +export function buildExtensionObservationV1( + rawSignal: ExtractedPageProductSignal, + observedAt: Date, + extensionVersion: string +): ExtensionObservationV1 | null { + const signal = sanitizeExtensionProductSignal(rawSignal); + const parsed = parseExtensionObservationV1( + { + schema_version: 1, + marketplace: signal.platform, + listing_url: signal.canonicalUrl, + external_item_id: signal.externalItemId, + external_shop_id: signal.externalShopId, + sku_id: signal.skuId ?? null, + variant_label: signal.selectedVariant, + price_vnd: signal.price, + list_price_vnd: signal.originalPrice, + stock_state: signal.stockState ?? 'unknown', + stock_qty: signal.stockQty ?? null, + observed_at: observedAt.toISOString(), + extractor: { + source: signal.extractorSource ?? 'dom', + extension_version: extensionVersion, + }, + }, + observedAt.getTime() + ); + return parsed.ok ? parsed.value : null; +} + export function escapeSafeExternalHref( rawUrl: string | null | undefined, fallbackUrl = 'https://rate.vn' diff --git a/apps/extension/src/test-extension-pipeline.ts b/apps/extension/src/test-extension-pipeline.ts index c641cfe40..40aada62e 100644 --- a/apps/extension/src/test-extension-pipeline.ts +++ b/apps/extension/src/test-extension-pipeline.ts @@ -5026,6 +5026,156 @@ async function runTests() { ); } + // --- DOS/Rate#1606: stock, SKU and the v1 observation (read, not sent) --- + { + const det = await import('./extractors/ecommerce-detector.js'); + const guards = await import('./security-guards.js'); + assert.equal( + det.stockStateFromAvailability('https://schema.org/InStock'), + 'in_stock' + ); + assert.equal( + det.stockStateFromAvailability('http://schema.org/OutOfStock'), + 'out_of_stock' + ); + assert.equal( + det.stockStateFromAvailability('LimitedAvailability'), + 'low_stock' + ); + assert.equal(det.stockStateFromAvailability('PreOrder'), 'unknown'); + assert.equal(det.stockStateFromAvailability('in stock'), 'in_stock'); + assert.equal(det.stockStateFromAvailability('blue'), null); + assert.equal(det.stockStateFromAvailability(42), null); + assert.deepEqual( + det.extractStockFromJsonLdOffers([ + { price: 1 }, + { + availability: 'https://schema.org/InStock', + inventoryLevel: { value: 7 }, + }, + ]), + { stockState: 'in_stock', stockQty: 7 } + ); + assert.deepEqual( + det.extractStockFromJsonLdOffers({ + availability: 'OutOfStock', + inventoryLevel: 9, + }), + { stockState: 'out_of_stock', stockQty: 0 } + ); + assert.deepEqual( + det.extractStockFromJsonLdOffers({ + availability: 'InStock', + inventoryLevel: -3, + }), + { stockState: 'in_stock', stockQty: null } + ); + assert.equal( + det.extractSkuId( + 'lazada', + 'https://www.lazada.vn/products/iphone-i123-s456.html', + null + ), + '456' + ); + assert.equal( + det.extractSkuId( + 'tiki', + 'https://tiki.vn/iphone-p123.html?spid=789', + 'X1' + ), + '789' + ); + assert.equal( + det.extractSkuId('shopee', 'https://shopee.vn/x-i.1.2', 'MDL-9'), + 'MDL-9' + ); + assert.equal( + det.extractSkuId('shopee', 'https://shopee.vn/x-i.1.2', 'a b'), + null + ); + assert.equal( + det.extractSkuId('tiki', 'https://tiki.vn/x-p1.html?spid=abc', null), + null + ); + + const signal = { + platform: 'shopee' as const, + canonicalUrl: 'https://shopee.vn/iphone-16-i.123.456', + externalItemId: '456', + externalShopId: '123', + title: 'iPhone 16 128GB', + selectedVariant: 'Đen', + price: 19_990_000, + originalPrice: 22_990_000, + ratingStar: 4.9, + ratingCount: 1000, + skuId: '789', + stockState: 'in_stock' as const, + stockQty: 5, + extractorSource: 'json_ld' as const, + }; + const at = new Date('2026-11-04T10:00:00Z'); + const obs = guards.buildExtensionObservationV1(signal, at, '2.1.0'); + assert.deepEqual(obs, { + schema_version: 1, + marketplace: 'shopee', + listing_url: 'https://shopee.vn/iphone-16-i.123.456', + external_item_id: '456', + external_shop_id: '123', + sku_id: '789', + variant_label: 'Đen', + price_vnd: 19_990_000, + list_price_vnd: 22_990_000, + stock_state: 'in_stock', + stock_qty: 5, + observed_at: '2026-11-04T10:00:00.000Z', + extractor: { source: 'json_ld', extension_version: '2.1.0' }, + }); + // No ratings in v1 (JOY approved dropping them). + assert.ok(obs && !('rating_star' in obs) && !('ratingStar' in obs)); + assert.equal( + guards.buildExtensionObservationV1( + { ...signal, price: null }, + at, + '2.1.0' + ), + null + ); + assert.equal( + guards.buildExtensionObservationV1( + { ...signal, externalItemId: null }, + at, + '2.1.0' + ), + null + ); + // The sanitizer keeps only valid v1 fields. + const dirty = guards.sanitizeExtensionProductSignal({ + ...signal, + skuId: 'a b', + stockState: 'lots' as never, + stockQty: -1, + extractorSource: 'api' as never, + }); + assert.equal(dirty.skuId, null); + assert.equal(dirty.stockState, 'unknown'); + assert.equal(dirty.stockQty, null); + assert.equal(dirty.extractorSource, 'dom'); + // Neither the opt-in signal nor the lookup carries the new fields yet. + const crowd = guards.buildCrowdIngestPayload(signal as never); + for (const key of ['skuId', 'stockState', 'stockQty', 'extractorSource']) { + assert.ok(!(key in crowd), key); + } + const workerSource = fs.readFileSync( + path.resolve(__dirname, 'background/service-worker.ts'), + 'utf8' + ); + assert.ok(!/stockState|skuId|stockQty/.test(workerSource)); + console.log( + ' ✅ [Stock, SKU and the v1 observation, not sent (#1606)] Verified' + ); + } console.log('🎉 All Rate Browser Extension MV3 v2 tests passed!'); } diff --git a/apps/extension/src/types.ts b/apps/extension/src/types.ts index 0328f57fb..e40ab530a 100644 --- a/apps/extension/src/types.ts +++ b/apps/extension/src/types.ts @@ -202,6 +202,9 @@ export interface ExtensionLookupData { } | null; } +export type ExtractedStockState = + 'in_stock' | 'low_stock' | 'out_of_stock' | 'unknown'; + export interface ExtractedPageProductSignal { platform: SupportedEcommercePlatform; canonicalUrl: string; @@ -213,6 +216,15 @@ export interface ExtractedPageProductSignal { originalPrice: number | null; ratingStar: number | null; ratingCount: number | null; + /** + * Read for ExtensionObservation v1 (DOS/Rate#1606) and not sent yet: + * neither the lookup nor the opt-in signal carries them. + */ + skuId?: string | null; + stockState?: ExtractedStockState; + stockQty?: number | null; + /** Where the title and price came from. */ + extractorSource?: 'json_ld' | 'dom' | 'mixed'; } export interface ExtensionSettings { diff --git a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts index 8524c91de..857419735 100644 --- a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts +++ b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts @@ -829,6 +829,235 @@ async function testMyReviewValues() { assert.equal(res.status, 502); } +async function testExtensionObservations() { + const submitterRoute = await import('../extension/submitter/route'); + const observationRoute = await import('../extension/observation/route'); + const obsMod = await import('@/server/rate/extension-observations'); + const SUBMITTER = 'cccccccc-1111-4222-8333-444444444444'; + const observation = { + schema_version: 1, + marketplace: 'shopee', + listing_url: 'https://shopee.vn/iphone-16-i.123.456', + external_item_id: '456', + external_shop_id: '123', + sku_id: '789', + variant_label: 'Đen', + price_vnd: 19_990_000, + list_price_vnd: 22_990_000, + stock_state: 'in_stock', + stock_qty: 5, + observed_at: new Date(Date.now() - 60_000).toISOString(), + extractor: { source: 'json_ld', extension_version: '2.1.0' }, + }; + const withIp = (req: Request, ip: string) => { + req.headers.set('x-forwarded-for', ip); + return req; + }; + const del = (body: unknown) => + new Request('http://localhost/api/v1/rate/extension/submitter', { + method: 'DELETE', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + + // Off by default: 404 and no database call on any route. + delete process.env.RATE_EXTENSION_OBSERVATIONS; + calls = []; + for (const res of [ + await submitterRoute.POST(post('/api/v1/rate/extension/submitter', {})), + await submitterRoute.DELETE(del({ submitterId: SUBMITTER })), + await observationRoute.POST( + post('/api/v1/rate/extension/observation', { + submitterId: SUBMITTER, + observation, + }) + ), + ]) { + assert.equal(res.status, 404); + assert.equal((await res.json()).code, 'NOT_ENABLED'); + } + assert.equal(calls.length, 0, 'a disabled route touches nothing'); + + process.env.RATE_EXTENSION_OBSERVATIONS = '1'; + const savedSalt = process.env.RATE_NETWORK_HASH_SALT; + try { + // On, but without the network hash salt: observations are refused, + // since the per-network limit would not hold. + delete process.env.RATE_NETWORK_HASH_SALT; + calls = []; + const noSalt = await observationRoute.POST( + post('/api/v1/rate/extension/observation', { + submitterId: SUBMITTER, + observation, + }) + ); + assert.equal(noSalt.status, 503); + assert.equal(calls.length, 0); + process.env.RATE_NETWORK_HASH_SALT = 'test-salt-0123456789'; + + // A submitter id comes from the RPC, as anon, never the service key. + calls = []; + rpcResponder = (name) => + name === 'register_extension_submitter' + ? { status: 200, json: SUBMITTER } + : { status: 500, json: { message: 'unexpected rpc' } }; + let res = await submitterRoute.POST( + post('/api/v1/rate/extension/submitter', {}) + ); + assert.equal(res.status, 200); + assert.deepEqual(await res.json(), { ok: true, submitterId: SUBMITTER }); + assert.match(res.headers.get('cache-control') ?? '', /no-store/); + assert.ok( + calls.every((c) => !JSON.stringify(c.headers).includes(SERVICE_KEY)) + ); + + // A valid observation goes to the RPC with a hash of the /24, not the IP. + calls = []; + rpcResponder = (name) => + name === 'submit_extension_observation' + ? { + status: 200, + json: { id: 1, status: 'quarantined', received_at: 'now' }, + } + : { status: 500, json: { message: 'unexpected rpc' } }; + res = await observationRoute.POST( + withIp( + post('/api/v1/rate/extension/observation', { + submitterId: SUBMITTER.toUpperCase(), + observation, + }), + '203.0.113.77' + ) + ); + assert.equal(res.status, 200); + assert.deepEqual(await res.json(), { ok: true, status: 'quarantined' }); + const [call] = rpcCalls('submit_extension_observation'); + assert.ok(call, 'submit_extension_observation must be called'); + const args = call.body as Record; + assert.equal(args.p_submitter_id, SUBMITTER); + assert.deepEqual(args.p_observation, observation); + assert.equal( + args.p_network_hash, + obsMod.networkHashFor('203.0.113.1', 'test-salt-0123456789') + ); + assert.match(String(args.p_network_hash), /^[0-9a-f]{64}$/); + assert.ok( + !JSON.stringify(calls).includes('203.0.113.77'), + 'the raw IP is never sent on' + ); + + // Bad input is refused before the database. + for (const [body, code, field] of [ + [{ submitterId: 'x', observation }, 'INVALID_SUBMITTER', undefined], + [ + { + submitterId: SUBMITTER, + observation: { ...observation, rating_star: 5 }, + }, + 'INVALID_OBSERVATION', + 'observation', + ], + [ + { + submitterId: SUBMITTER, + observation: { ...observation, listing_url: 'https://evil.com/x' }, + }, + 'INVALID_OBSERVATION', + 'listing_url', + ], + ] as const) { + calls = []; + res = await observationRoute.POST( + post('/api/v1/rate/extension/observation', body) + ); + assert.equal(res.status, 400); + const json = await res.json(); + assert.equal(json.code, code); + if (field) assert.equal(json.field, field); + assert.equal(calls.length, 0); + } + + // Database answers map to fixed codes, never a faked success. + for (const [error, status, code] of [ + [{ code: '22023', details: 'price_vnd' }, 400, 'INVALID_OBSERVATION'], + [{ code: 'P0002', details: 'submitter_id' }, 404, 'UNKNOWN_SUBMITTER'], + [{ code: '54000' }, 429, 'RATE_LIMIT_EXCEEDED'], + [ + { code: 'PGRST202', message: 'Could not find' }, + 503, + 'OBSERVATIONS_UNAVAILABLE', + ], + [{ code: 'XX000', message: 'boom' }, 502, 'PERSIST_FAILED'], + ] as const) { + rpcResponder = () => ({ + status: error.code === 'PGRST202' ? 404 : 400, + json: error, + }); + res = await observationRoute.POST( + post('/api/v1/rate/extension/observation', { + submitterId: SUBMITTER, + observation, + }) + ); + assert.equal(res.status, status, error.code); + const json = await res.json(); + assert.equal(json.code, code, error.code); + if (error.code === '22023') assert.equal(json.field, 'price_vnd'); + } + + // Deleting answers the same for any id and calls the delete RPC. + calls = []; + rpcResponder = (name) => + name === 'delete_extension_submitter' + ? { status: 200, json: 0 } + : { status: 500, json: { message: 'unexpected rpc' } }; + res = await submitterRoute.DELETE(del({ submitterId: SUBMITTER })); + assert.equal(res.status, 200); + assert.deepEqual(await res.json(), { ok: true }); + assert.deepEqual(rpcCalls('delete_extension_submitter')[0]?.body, { + p_submitter_id: SUBMITTER, + }); + res = await submitterRoute.DELETE(del({ submitterId: 'nope' })); + assert.equal(res.status, 400); + + // Network ranges: /24 for IPv4, /48 for IPv6, nothing otherwise. + assert.equal(obsMod.networkPrefixOf('203.0.113.77'), '203.0.113.0/24'); + assert.equal( + obsMod.networkPrefixOf('2001:db8:abcd:12::1'), + '2001:db8:abcd::/48' + ); + assert.equal( + obsMod.networkPrefixOf('2001:0db8:0000::1'), + '2001:db8:0::/48' + ); + assert.equal(obsMod.networkPrefixOf('::1'), '0:0:0::/48'); + for (const bad of [ + 'anon', + '999.1.1.1', + '1.2.3', + 'a::b::c', + 'fe80::1%eth0', + '', + ]) { + assert.equal(obsMod.networkPrefixOf(bad), null, bad); + } + assert.equal(obsMod.networkHashFor('203.0.113.77', 'short'), null); + assert.equal(obsMod.networkHashFor('anon', 'test-salt-0123456789'), null); + assert.equal( + obsMod.networkHashFor('203.0.113.5', 'test-salt-0123456789'), + obsMod.networkHashFor('203.0.113.200', 'test-salt-0123456789') + ); + assert.notEqual( + obsMod.networkHashFor('203.0.113.5', 'test-salt-0123456789'), + obsMod.networkHashFor('203.0.114.5', 'test-salt-0123456789') + ); + } finally { + delete process.env.RATE_EXTENSION_OBSERVATIONS; + if (savedSalt === undefined) delete process.env.RATE_NETWORK_HASH_SALT; + else process.env.RATE_NETWORK_HASH_SALT = savedSalt; + } +} + async function testWatchlist() { const { POST } = await import('../watchlist/route'); type Req = Parameters[0]; @@ -2120,6 +2349,7 @@ async function main() { await testReviewsV2(); await testReviewsByIdPublishedOnly(); await testMyReviewValues(); + await testExtensionObservations(); await testWatchlist(); await testCookieSessions(); await testFilterBoundsAndWatchlistPrices(); diff --git a/apps/web/src/app/api/v1/rate/extension/observation/route.ts b/apps/web/src/app/api/v1/rate/extension/observation/route.ts new file mode 100644 index 000000000..6157ccfec --- /dev/null +++ b/apps/web/src/app/api/v1/rate/extension/observation/route.ts @@ -0,0 +1,109 @@ +import { parseExtensionObservationV1 } from '@repo/shared-data/rules'; +import { + BoundedSlidingWindowRateLimiter, + extractClientRateLimitKey, + payloadTooLargeResponse, + rateLimitExceededResponse, + readBoundedJsonBody, +} from '@/server/rate/api-handler'; +import { + EXTENSION_OBSERVATION_HEADERS, + callObservationRpc, + hasNetworkHashSalt, + isExtensionObservationsEnabled, + networkHashFor, + notEnabledResponse, + observationResponse, + parseSubmitterId, + rpcFailureResponse, +} from '@/server/rate/extension-observations'; +import { createLogger, serializeError } from '@/lib/logger'; + +const log = createLogger('api.extension.observation'); + +// Best-effort per-instance brake; the RPC holds the per-submitter and +// per-network limits. +const observationLimiter = new BoundedSlidingWindowRateLimiter({ + windowMs: 60_000, + maxRequests: 60, + maxBuckets: 2_000, +}); + +export async function OPTIONS() { + return new Response(null, { + status: 204, + headers: EXTENSION_OBSERVATION_HEADERS, + }); +} + +/** + * POST /api/v1/rate/extension/observation { submitterId, observation } + * (DOS/Rate#1606). The observation must be an ExtensionObservation v1; it is + * stored in the quarantine, never as a public price. The raw IP is not sent + * on: only a salted hash of its network range goes to the RPC. + */ +export async function POST(request: Request) { + if (!isExtensionObservationsEnabled()) return notEnabledResponse(); + if (!hasNetworkHashSalt()) { + return observationResponse(503, { + ok: false, + code: 'OBSERVATIONS_UNAVAILABLE', + error: 'Extension observations are not available yet.', + }); + } + const clientIp = extractClientRateLimitKey(request.headers); + if (!observationLimiter.allow(clientIp)) { + return rateLimitExceededResponse( + 'Too many observations; try again in a minute.', + EXTENSION_OBSERVATION_HEADERS + ); + } + const body = await readBoundedJsonBody(request, 8 * 1024); + if (body.tooLarge) + return payloadTooLargeResponse(EXTENSION_OBSERVATION_HEADERS); + const raw = + body.value && typeof body.value === 'object' && !Array.isArray(body.value) + ? (body.value as { submitterId?: unknown; observation?: unknown }) + : {}; + const submitterId = parseSubmitterId(raw.submitterId); + if (!submitterId) { + return observationResponse(400, { + ok: false, + code: 'INVALID_SUBMITTER', + error: 'Provide submitterId as a uuid.', + }); + } + const parsed = parseExtensionObservationV1(raw.observation); + if (!parsed.ok) { + return observationResponse(400, { + ok: false, + code: 'INVALID_OBSERVATION', + error: 'The observation was refused.', + field: parsed.field, + }); + } + try { + const { data, error } = await callObservationRpc( + 'submit_extension_observation', + { + p_submitter_id: submitterId, + p_observation: parsed.value, + p_network_hash: networkHashFor(clientIp), + } + ); + if (error) return rpcFailureResponse(error); + const stored = + data && typeof data === 'object' && !Array.isArray(data) + ? (data as { status?: unknown }) + : null; + return observationResponse(200, { + ok: true, + status: stored?.status === 'quarantined' ? 'quarantined' : 'received', + }); + } catch (err) { + log.error('submit_extension_observation failed', { + error: serializeError(err), + }); + return rpcFailureResponse({}); + } +} diff --git a/apps/web/src/app/api/v1/rate/extension/submitter/route.ts b/apps/web/src/app/api/v1/rate/extension/submitter/route.ts new file mode 100644 index 000000000..35f18738f --- /dev/null +++ b/apps/web/src/app/api/v1/rate/extension/submitter/route.ts @@ -0,0 +1,113 @@ +import { + BoundedSlidingWindowRateLimiter, + extractClientRateLimitKey, + payloadTooLargeResponse, + rateLimitExceededResponse, + readBoundedJsonBody, +} from '@/server/rate/api-handler'; +import { + EXTENSION_OBSERVATION_HEADERS, + callObservationRpc, + isExtensionObservationsEnabled, + notEnabledResponse, + observationResponse, + parseSubmitterId, + rpcFailureResponse, +} from '@/server/rate/extension-observations'; +import { createLogger, serializeError } from '@/lib/logger'; + +const log = createLogger('api.extension.submitter'); + +// Best-effort per-instance brakes; the RPCs hold the global ones. +const registerLimiter = new BoundedSlidingWindowRateLimiter({ + windowMs: 60_000, + maxRequests: 10, + maxBuckets: 2_000, +}); +const deleteLimiter = new BoundedSlidingWindowRateLimiter({ + windowMs: 60_000, + maxRequests: 10, + maxBuckets: 2_000, +}); + +export async function OPTIONS() { + return new Response(null, { + status: 204, + headers: EXTENSION_OBSERVATION_HEADERS, + }); +} + +/** + * POST /api/v1/rate/extension/submitter: a new pseudonymous submitter id for + * an install that opted in (DOS/Rate#1606). No body; the server creates the + * id and records when. + */ +export async function POST(request: Request) { + if (!isExtensionObservationsEnabled()) return notEnabledResponse(); + if (!registerLimiter.allow(extractClientRateLimitKey(request.headers))) { + return rateLimitExceededResponse( + 'Too many requests; try again in a minute.', + EXTENSION_OBSERVATION_HEADERS + ); + } + try { + const { data, error } = await callObservationRpc( + 'register_extension_submitter' + ); + if (error) return rpcFailureResponse(error); + const submitterId = parseSubmitterId(data); + if (!submitterId) { + return observationResponse(502, { + ok: false, + code: 'PERSIST_FAILED', + error: 'Could not create a submitter id.', + }); + } + return observationResponse(200, { ok: true, submitterId }); + } catch (err) { + log.error('register_extension_submitter failed', { + error: serializeError(err), + }); + return rpcFailureResponse({}); + } +} + +/** + * DELETE /api/v1/rate/extension/submitter { submitterId }: deletes the + * submitter and every observation it sent (the data subject's right to + * erasure). The answer is the same whether or not the id existed. + */ +export async function DELETE(request: Request) { + if (!isExtensionObservationsEnabled()) return notEnabledResponse(); + if (!deleteLimiter.allow(extractClientRateLimitKey(request.headers))) { + return rateLimitExceededResponse( + 'Too many requests; try again in a minute.', + EXTENSION_OBSERVATION_HEADERS + ); + } + const body = await readBoundedJsonBody(request, 1024); + if (body.tooLarge) + return payloadTooLargeResponse(EXTENSION_OBSERVATION_HEADERS); + const submitterId = parseSubmitterId( + (body.value as { submitterId?: unknown } | null)?.submitterId + ); + if (!submitterId) { + return observationResponse(400, { + ok: false, + code: 'INVALID_SUBMITTER', + error: 'Provide submitterId as a uuid.', + }); + } + try { + const { error } = await callObservationRpc('delete_extension_submitter', { + p_submitter_id: submitterId, + }); + if (error) return rpcFailureResponse(error); + return observationResponse(200, { ok: true }); + } catch (err) { + log.error('delete_extension_submitter failed', { + error: serializeError(err), + }); + return rpcFailureResponse({}); + } +} diff --git a/apps/web/src/server/rate/extension-observations.ts b/apps/web/src/server/rate/extension-observations.ts new file mode 100644 index 000000000..61252da5f --- /dev/null +++ b/apps/web/src/server/rate/extension-observations.ts @@ -0,0 +1,174 @@ +import { createHmac } from 'node:crypto'; +import { NextResponse } from 'next/server'; +import { getRateClient } from '@/lib/rate-api'; +import { isMissingRpcError } from '@/server/rate/request-auth'; +import { rateLimitExceededResponse } from '@/server/rate/api-handler'; + +/** + * ExtensionObservation v1 ingest (DOS/Rate#1606, M2 of + * Docs/Proposals/2026-10-28-extension-data-source.md). An opted-in install + * gets a pseudonymous submitter id and sends observations into the + * quarantine table, through the narrow RPCs of + * Docs/Proposals/2026-11-04-dos-me-extension-observations.sql. Nothing + * reaches public prices; Rate-Services verifies (M3). + * + * Off until the table exists: every route answers 404 unless + * RATE_EXTENSION_OBSERVATIONS is "1", and then makes no database call. + */ + +export const EXTENSION_OBSERVATION_HEADERS: Record = { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'POST, DELETE, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type, X-Rate-Extension-Version', + 'Cache-Control': 'no-store', +}; + +export function isExtensionObservationsEnabled(): boolean { + return process.env.RATE_EXTENSION_OBSERVATIONS === '1'; +} + +/** + * The per-network limit needs the salt: without it every request's + * network_hash is null and fresh submitter ids walk around the + * per-submitter limit. Observations are refused until it is set. + */ +export function hasNetworkHashSalt( + salt = process.env.RATE_NETWORK_HASH_SALT +): boolean { + return typeof salt === 'string' && salt.length >= 16; +} + +export function observationResponse( + status: number, + body: Record +): NextResponse { + return NextResponse.json(body, { + status, + headers: EXTENSION_OBSERVATION_HEADERS, + }); +} + +export const notEnabledResponse = () => + observationResponse(404, { + ok: false, + code: 'NOT_ENABLED', + error: 'Extension observations are not enabled.', + }); + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +export function parseSubmitterId(raw: unknown): string | null { + return typeof raw === 'string' && UUID_RE.test(raw) + ? raw.toLowerCase() + : null; +} + +/** + * The network range of an IP: the /24 of IPv4, the /48 of IPv6; null for + * anything else (no address, a placeholder, a malformed value). + */ +export function networkPrefixOf(ip: string): string | null { + const v4 = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(ip); + if (v4) { + const parts = v4.slice(1, 5).map(Number); + if (parts.some((p) => p > 255)) return null; + return `${parts[0]}.${parts[1]}.${parts[2]}.0/24`; + } + if (!/^[0-9a-f:]{2,39}$/i.test(ip) || !ip.includes(':')) return null; + const halves = ip.toLowerCase().split('::'); + if (halves.length > 2) return null; + const head = halves[0] ? halves[0].split(':') : []; + const tail = halves.length === 2 && halves[1] ? halves[1].split(':') : []; + const missing = 8 - head.length - tail.length; + if (halves.length === 1 ? missing !== 0 : missing < 1) return null; + const groups = [...head, ...Array(missing).fill('0'), ...tail]; + if (groups.some((g) => !/^[0-9a-f]{1,4}$/.test(g))) return null; + return `${groups + .slice(0, 3) + .map((g) => g.replace(/^0+(?=.)/, '')) + .join(':')}::/48`; +} + +/** + * HMAC-SHA256 of the network range with RATE_NETWORK_HASH_SALT, as lowercase + * hex: lets Rate-Services tell independent submitters apart without the raw + * IP ever being stored. Null without a salt of at least 16 characters or + * without a usable address. + */ +export function networkHashFor( + ip: string, + salt = process.env.RATE_NETWORK_HASH_SALT +): string | null { + if (typeof salt !== 'string' || salt.length < 16) return null; + const prefix = networkPrefixOf(ip); + return prefix + ? createHmac('sha256', salt).update(prefix).digest('hex') + : null; +} + +interface LooseRpcClient { + rpc: ( + fn: string, + args?: Record + ) => PromiseLike<{ + data: unknown; + error: { code?: string; message?: string; details?: string } | null; + }>; +} + +/** Calls one of the M2 RPCs as anon; their types land with the migration. */ +export function callObservationRpc( + fn: + | 'register_extension_submitter' + | 'submit_extension_observation' + | 'delete_extension_submitter', + args?: Record +) { + return (getRateClient() as unknown as LooseRpcClient).rpc(fn, args); +} + +/** An RPC failure as an answer: never a success, never the raw message. */ +export function rpcFailureResponse(error: { + code?: string; + details?: string; +}): NextResponse { + if (isMissingRpcError(error)) { + return observationResponse(503, { + ok: false, + code: 'OBSERVATIONS_UNAVAILABLE', + error: 'Extension observations are not available yet.', + }); + } + if (error.code === '54000') { + return rateLimitExceededResponse( + 'Too many observations; try again later.', + EXTENSION_OBSERVATION_HEADERS + ); + } + if (error.code === 'P0002') { + return observationResponse(404, { + ok: false, + code: 'UNKNOWN_SUBMITTER', + error: 'Unknown submitter.', + }); + } + if (error.code === '22023') { + const field = + typeof error.details === 'string' && + /^[a-z_]{1,32}(?:\.[a-z_]{1,32})?$/.test(error.details) + ? error.details + : null; + return observationResponse(400, { + ok: false, + code: 'INVALID_OBSERVATION', + error: 'The observation was refused.', + field, + }); + } + return observationResponse(502, { + ok: false, + code: 'PERSIST_FAILED', + error: 'Could not store the observation.', + }); +} diff --git a/packages/shared-data/src/__tests__/shared-data.test.ts b/packages/shared-data/src/__tests__/shared-data.test.ts index 34fe87ffc..9157f0345 100644 --- a/packages/shared-data/src/__tests__/shared-data.test.ts +++ b/packages/shared-data/src/__tests__/shared-data.test.ts @@ -2322,6 +2322,109 @@ function testReviewDepth(): void { console.log(' ✅ review depth (axis C) over the whole review text (#1581)'); } +function testExtensionObservation(): void { + const now = Date.parse('2026-11-04T10:00:00Z'); + const valid = { + schema_version: 1, + marketplace: 'shopee', + listing_url: 'https://shopee.vn/iphone-16-i.123.456', + external_item_id: '456', + external_shop_id: '123', + sku_id: '789', + variant_label: 'Đen, 128GB', + price_vnd: 19_990_000, + list_price_vnd: 22_990_000, + stock_state: 'in_stock', + stock_qty: 12, + observed_at: '2026-11-04T09:59:00Z', + extractor: { source: 'json_ld', extension_version: '2.1.0' }, + }; + const parse = (o: unknown) => rules.parseExtensionObservationV1(o, now); + const ok = parse(valid); + assert.ok(ok.ok); + assert.deepEqual(ok.ok && ok.value, valid); + // Nullable fields may be null, but must be present. + assert.ok( + parse({ + ...valid, + external_shop_id: null, + sku_id: null, + variant_label: null, + list_price_vnd: null, + stock_qty: null, + stock_state: 'unknown', + }).ok + ); + const { sku_id: _dropped, ...missing } = valid; + void _dropped; + const cases: Array<[Record | unknown, string]> = [ + [null, 'observation'], + [[], 'observation'], + [{ ...valid, rating_star: 5 }, 'observation'], + [missing, 'sku_id'], + [{ ...valid, schema_version: 2 }, 'schema_version'], + [{ ...valid, marketplace: 'amazon' }, 'marketplace'], + [ + { ...valid, listing_url: 'https://shopee.vn/x?utm_source=a' }, + 'listing_url', + ], + [{ ...valid, listing_url: 'https://shopee.vn/x#a' }, 'listing_url'], + [{ ...valid, listing_url: 'http://shopee.vn/x' }, 'listing_url'], + [{ ...valid, listing_url: 'https://evil.com/shopee.vn' }, 'listing_url'], + [{ ...valid, listing_url: 'https://shopee.vn.evil.com/x' }, 'listing_url'], + [{ ...valid, listing_url: 'https://u:p@shopee.vn/x' }, 'listing_url'], + [ + { ...valid, listing_url: `https://shopee.vn/${'a'.repeat(2048)}` }, + 'listing_url', + ], + [{ ...valid, external_item_id: '' }, 'external_item_id'], + [{ ...valid, external_item_id: 'a b' }, 'external_item_id'], + [{ ...valid, external_shop_id: 'x'.repeat(65) }, 'external_shop_id'], + [{ ...valid, sku_id: 7 }, 'sku_id'], + [{ ...valid, variant_label: '' }, 'variant_label'], + [{ ...valid, variant_label: 'a\u0000b' }, 'variant_label'], + [{ ...valid, price_vnd: 999 }, 'price_vnd'], + [{ ...valid, price_vnd: 1.5e3 + 0.5 }, 'price_vnd'], + [{ ...valid, price_vnd: '19990000' }, 'price_vnd'], + [{ ...valid, list_price_vnd: 19_000_000 }, 'list_price_vnd'], + [{ ...valid, stock_state: 'many' }, 'stock_state'], + [{ ...valid, stock_qty: -1 }, 'stock_qty'], + [{ ...valid, stock_qty: 1_000_001 }, 'stock_qty'], + [{ ...valid, observed_at: '2026-11-04T10:11:00Z' }, 'observed_at'], + [{ ...valid, observed_at: '2026-11-03T09:59:00Z' }, 'observed_at'], + [{ ...valid, observed_at: 'yesterday' }, 'observed_at'], + [{ ...valid, extractor: 'json_ld' }, 'extractor'], + [{ ...valid, extractor: { ...valid.extractor, user: 'x' } }, 'extractor'], + [ + { ...valid, extractor: { ...valid.extractor, source: 'api' } }, + 'extractor.source', + ], + [ + { ...valid, extractor: { ...valid.extractor, extension_version: '2 1' } }, + 'extractor.extension_version', + ], + ]; + for (const [input, field] of cases) { + const result = parse(input); + assert.equal(result.ok, false, JSON.stringify(input).slice(0, 80)); + assert.equal( + !result.ok && result.field, + field, + JSON.stringify(input).slice(0, 80) + ); + } + // Every marketplace accepts its own hosts and refuses another's. + assert.ok(rules.isExtensionMarketplaceHost('lazada', 'www.lazada.vn')); + assert.ok(rules.isExtensionMarketplaceHost('tgdd', 'www.thegioididong.com')); + assert.ok(rules.isExtensionMarketplaceHost('tiktok', 'shop.tiktok.com')); + assert.ok(!rules.isExtensionMarketplaceHost('tiktok', 'm.tiktok.com')); + assert.ok(!rules.isExtensionMarketplaceHost('lazada', 'notlazada.vn')); + assert.ok(!rules.isExtensionMarketplaceHost('shopee', 'tiki.vn')); + for (const m of rules.EXTENSION_MARKETPLACES) { + assert.ok(rules.EXTENSION_MARKETPLACE_HOSTS[m].length > 0, m); + } +} + function testReviewValue(): void { const phone = rules.getRatingCriteria('smartphones'); const allRated: Record = Object.fromEntries( @@ -2686,6 +2789,8 @@ function main(): void { console.log( ' ✅ [13/13] Review value score and weight factor (DOS/Rate#1547) verified' ); + testExtensionObservation(); + console.log(' ✅ [14/14] ExtensionObservation v1 (DOS/Rate#1606) verified'); console.log('🎉 All @repo/shared-data tests passed!'); } diff --git a/packages/shared-data/src/rules/extension-observation.ts b/packages/shared-data/src/rules/extension-observation.ts new file mode 100644 index 000000000..d0d112d21 --- /dev/null +++ b/packages/shared-data/src/rules/extension-observation.ts @@ -0,0 +1,290 @@ +/** + * ExtensionObservation v1 (DOS/Rate#1606, milestone M2 of + * Docs/Proposals/2026-10-28-extension-data-source.md): the minimal price, + * stock and SKU observation an opted-in browser extension sends. It goes to + * a quarantine table and never to public prices directly. + * + * The rules here are repeated by rate.submit_extension_observation in the + * DOS-Me migration (Docs/Proposals/2026-11-04-dos-me-extension-observations.sql); + * a change to one needs the other. Plain functions only, no zod, so the + * extension and client code can import it. + */ + +export const EXTENSION_OBSERVATION_VERSION = 1; + +const EXTENSION_MARKETPLACES = [ + 'shopee', + 'lazada', + 'tiki', + 'tiktok', + 'cellphones', + 'tgdd', + 'fptshop', + 'hoangha', + 'didongviet', + 'phongvu', +] as const; + +export type ExtensionMarketplace = (typeof EXTENSION_MARKETPLACES)[number]; + +const EXTENSION_STOCK_STATES = [ + 'in_stock', + 'low_stock', + 'out_of_stock', + 'unknown', +] as const; + +export type ExtensionStockState = (typeof EXTENSION_STOCK_STATES)[number]; + +const EXTENSION_EXTRACTOR_SOURCES = ['json_ld', 'dom', 'mixed'] as const; + +/** The hosts each marketplace's listings live on; `.x` also allows subdomains. */ +const EXTENSION_MARKETPLACE_HOSTS: Readonly< + Record +> = { + shopee: ['shopee.vn'], + lazada: ['lazada.vn', '.lazada.vn'], + tiki: ['tiki.vn'], + tiktok: ['www.tiktok.com', 'shop.tiktok.com'], + cellphones: ['cellphones.com.vn'], + tgdd: ['thegioididong.com', '.thegioididong.com'], + fptshop: ['fptshop.com.vn'], + hoangha: ['hoanghamobile.com'], + didongviet: ['didongviet.vn'], + phongvu: ['phongvu.vn'], +}; + +export const EXTENSION_OBSERVATION_LIMITS = { + urlMax: 2048, + idMax: 64, + variantMax: 120, + priceMin: 1_000, + priceMax: 2_000_000_000, + stockQtyMax: 1_000_000, + extensionVersionMax: 20, + /** observed_at may run ahead of the server clock by this much. */ + futureSkewMs: 10 * 60 * 1000, + /** observed_at may be at most this old. */ + maxAgeMs: 24 * 60 * 60 * 1000, +} as const; + +export interface ExtensionObservationV1 { + schema_version: 1; + marketplace: ExtensionMarketplace; + listing_url: string; + external_item_id: string; + external_shop_id: string | null; + sku_id: string | null; + variant_label: string | null; + price_vnd: number; + list_price_vnd: number | null; + stock_state: ExtensionStockState; + stock_qty: number | null; + observed_at: string; + extractor: { + source: (typeof EXTENSION_EXTRACTOR_SOURCES)[number]; + extension_version: string; + }; +} + +export type ExtensionObservationField = + | keyof ExtensionObservationV1 + | 'observation' + | 'extractor.source' + | 'extractor.extension_version'; + +export type ExtensionObservationResult = + | { ok: true; value: ExtensionObservationV1 } + | { ok: false; field: ExtensionObservationField }; + +const KEYS: readonly (keyof ExtensionObservationV1)[] = [ + 'schema_version', + 'marketplace', + 'listing_url', + 'external_item_id', + 'external_shop_id', + 'sku_id', + 'variant_label', + 'price_vnd', + 'list_price_vnd', + 'stock_state', + 'stock_qty', + 'observed_at', + 'extractor', +]; + +const ID_RE = /^[A-Za-z0-9_.-]+$/; +const VERSION_RE = /^[0-9A-Za-z.+-]+$/; +// eslint-disable-next-line no-control-regex +const CONTROL_RE = /[\u0000-\u001f\u007f-\u009f]/; + +const isRecord = (v: unknown): v is Record => + typeof v === 'object' && v !== null && !Array.isArray(v); + +const has = (list: readonly T[], v: unknown): v is T => + typeof v === 'string' && (list as readonly string[]).includes(v); + +function idOk(v: unknown): v is string { + return ( + typeof v === 'string' && + v.length >= 1 && + v.length <= EXTENSION_OBSERVATION_LIMITS.idMax && + ID_RE.test(v) + ); +} + +const nullableId = (v: unknown) => v === null || idOk(v); + +const priceOk = (v: unknown): v is number => + typeof v === 'number' && + Number.isInteger(v) && + v >= EXTENSION_OBSERVATION_LIMITS.priceMin && + v <= EXTENSION_OBSERVATION_LIMITS.priceMax; + +/** True when `host` is one of the marketplace's listing hosts. */ +export function isExtensionMarketplaceHost( + marketplace: ExtensionMarketplace, + host: string +): boolean { + const h = host.toLowerCase(); + return EXTENSION_MARKETPLACE_HOSTS[marketplace].some((allowed) => + allowed.startsWith('.') ? h.endsWith(allowed) : h === allowed + ); +} + +function listingUrlOk(v: unknown, marketplace: ExtensionMarketplace): boolean { + if ( + typeof v !== 'string' || + v.length > EXTENSION_OBSERVATION_LIMITS.urlMax || + v.includes('?') || + v.includes('#') || + CONTROL_RE.test(v) + ) { + return false; + } + try { + const url = new URL(v); + return ( + url.protocol === 'https:' && + !url.username && + !url.password && + !url.port && + isExtensionMarketplaceHost(marketplace, url.hostname) + ); + } catch { + return false; + } +} + +/** + * Checks an observation against v1 at server time `nowMs`. The first field + * that breaks a rule is named, the way the RPC names it in DETAIL. + */ +export function parseExtensionObservationV1( + raw: unknown, + nowMs: number = Date.now() +): ExtensionObservationResult { + const fail = (field: ExtensionObservationField) => + ({ ok: false, field }) as const; + if (!isRecord(raw)) return fail('observation'); + for (const key of Object.keys(raw)) { + if (!(KEYS as readonly string[]).includes(key)) return fail('observation'); + } + if (raw.schema_version !== EXTENSION_OBSERVATION_VERSION) { + return fail('schema_version'); + } + const marketplace = raw.marketplace; + if (!has(EXTENSION_MARKETPLACES, marketplace)) return fail('marketplace'); + if (!listingUrlOk(raw.listing_url, marketplace)) return fail('listing_url'); + if (!idOk(raw.external_item_id)) return fail('external_item_id'); + if (!nullableId(raw.external_shop_id)) return fail('external_shop_id'); + if (!nullableId(raw.sku_id)) return fail('sku_id'); + const variant = raw.variant_label; + if ( + variant !== null && + (typeof variant !== 'string' || + variant.length < 1 || + variant.length > EXTENSION_OBSERVATION_LIMITS.variantMax || + CONTROL_RE.test(variant)) + ) { + return fail('variant_label'); + } + if (!priceOk(raw.price_vnd)) return fail('price_vnd'); + const listPrice = raw.list_price_vnd; + if ( + listPrice !== null && + (!priceOk(listPrice) || listPrice < raw.price_vnd) + ) { + return fail('list_price_vnd'); + } + if (!has(EXTENSION_STOCK_STATES, raw.stock_state)) return fail('stock_state'); + const qty = raw.stock_qty; + if ( + qty !== null && + (typeof qty !== 'number' || + !Number.isInteger(qty) || + qty < 0 || + qty > EXTENSION_OBSERVATION_LIMITS.stockQtyMax) + ) { + return fail('stock_qty'); + } + const observedMs = + typeof raw.observed_at === 'string' && raw.observed_at.length <= 40 + ? Date.parse(raw.observed_at) + : Number.NaN; + if ( + !Number.isFinite(observedMs) || + observedMs > nowMs + EXTENSION_OBSERVATION_LIMITS.futureSkewMs || + observedMs < nowMs - EXTENSION_OBSERVATION_LIMITS.maxAgeMs + ) { + return fail('observed_at'); + } + const extractor = raw.extractor; + if (!isRecord(extractor)) return fail('extractor'); + for (const key of Object.keys(extractor)) { + if (key !== 'source' && key !== 'extension_version') + return fail('extractor'); + } + if (!has(EXTENSION_EXTRACTOR_SOURCES, extractor.source)) { + return fail('extractor.source'); + } + const version = extractor.extension_version; + if ( + typeof version !== 'string' || + version.length < 1 || + version.length > EXTENSION_OBSERVATION_LIMITS.extensionVersionMax || + !VERSION_RE.test(version) + ) { + return fail('extractor.extension_version'); + } + return { + ok: true, + value: { + schema_version: 1, + marketplace, + listing_url: raw.listing_url as string, + external_item_id: raw.external_item_id as string, + external_shop_id: raw.external_shop_id as string | null, + sku_id: raw.sku_id as string | null, + variant_label: variant as string | null, + price_vnd: raw.price_vnd as number, + list_price_vnd: listPrice as number | null, + stock_state: raw.stock_state as ExtensionStockState, + stock_qty: qty as number | null, + observed_at: raw.observed_at as string, + extractor: { + source: extractor.source, + extension_version: version, + }, + }, + }; +} + +// Exported here: an `export const` holding strings would read as an import +// to the rules bundle check in shared-data.test.ts. +export { + EXTENSION_MARKETPLACES, + EXTENSION_STOCK_STATES, + EXTENSION_EXTRACTOR_SOURCES, + EXTENSION_MARKETPLACE_HOSTS, +}; diff --git a/packages/shared-data/src/rules/index.ts b/packages/shared-data/src/rules/index.ts index b99154c9e..909c2a3a7 100644 --- a/packages/shared-data/src/rules/index.ts +++ b/packages/shared-data/src/rules/index.ts @@ -16,3 +16,4 @@ export * from './review-form.js'; export * from './product-points.js'; export * from './review-depth.js'; export * from './review-value.js'; +export * from './extension-observation.js'; From d19083ac00645e6296031283025b5dd751eaab78 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:05:03 +0700 Subject: [PATCH 2/4] fix(extension,web): m2 review fixes, ingest key and variant-true stock Web: - The RPCs now take RATE_EXTENSION_INGEST_KEY (checked by sha256 in the database), so nobody can call them around these routes with the anon key. Writes need the key and the salt, else 503 before the database. - Register sends the network hash too. An address with no range hashes to one shared "unknown" bucket instead of skipping the limit. IPv4-mapped IPv6 maps to its /24. - Web page origins are refused (403); CORS goes only to extension origins; bodies must be application/json (415). OPTIONS is 404 while off. RPC refusals log their code; 54000 answers Retry-After 3600; 42501 is a 503. Shared rules: listing_url and observed_at follow the RPC exactly (raw https prefix, lowercase host, no port, user, ';' or space; ISO 8601 with a zone, offsets as the RPC takes them); lengths count code points; unpaired surrogates are refused. Extension: - Stock and SKU come from the selected variant's own offer: the only offer, or the one whose sku or name matches the selected variant; never another variant's or the parent's when several exist. The meta fallback reads head meta only. - Availability: schema: prefixes, arrays, Reserved and pre-order forms. A quantity is kept only when it agrees with the state. - extractorSource follows price, list price and stock. A long variant label is cut to 120 for v1. skuId uses the external id sanitizer. - buildLookupParams carries the lookup query; tests pin the exact lookup query and opt-in payload, and cover a page end to end. Refs #1606 Co-Authored-By: Claude Opus 5.5 --- .../src/background/service-worker.ts | 19 +- .../src/extractors/ecommerce-detector.ts | 171 ++++++++++++------ apps/extension/src/security-guards.ts | 49 ++++- apps/extension/src/test-extension-pipeline.ts | 143 ++++++++++++++- .../api/v1/rate/__tests__/write-paths.test.ts | 104 +++++++++-- .../v1/rate/extension/observation/route.ts | 44 ++--- .../api/v1/rate/extension/submitter/route.ts | 59 +++--- .../src/server/rate/extension-observations.ts | 155 ++++++++++++---- .../src/__tests__/shared-data.test.ts | 38 ++++ .../src/rules/extension-observation.ts | 73 +++++--- 10 files changed, 656 insertions(+), 199 deletions(-) diff --git a/apps/extension/src/background/service-worker.ts b/apps/extension/src/background/service-worker.ts index 0ba4c584a..2a4753f49 100644 --- a/apps/extension/src/background/service-worker.ts +++ b/apps/extension/src/background/service-worker.ts @@ -11,6 +11,7 @@ import { } from '../extractors/ecommerce-detector.js'; import { buildCrowdIngestPayload, + buildLookupParams, DEFAULT_API_BASE_URL, isPrivilegedExtensionSender, normalizeAllowedApiBaseUrl, @@ -116,24 +117,8 @@ async function fetchExtensionLookup( rawSignal: ExtractedPageProductSignal, mode: 'pill' | 'full' ): Promise { - const signal = sanitizeExtensionProductSignal(rawSignal); const settings = await getSettings(); - const params = new URLSearchParams(); - params.set('platform', signal.platform); - params.set('url', signal.canonicalUrl); - params.set('title', signal.title); - params.set('mode', mode); - if (signal.externalItemId) params.set('externalId', signal.externalItemId); - if (signal.externalShopId) params.set('shopId', signal.externalShopId); - if (signal.selectedVariant) { - params.set('selectedVariant', signal.selectedVariant); - } - if (signal.price && signal.price > 0) { - params.set('price', String(signal.price)); - } - if (signal.originalPrice && signal.originalPrice > 0) { - params.set('originalPrice', String(signal.originalPrice)); - } + const params = buildLookupParams(rawSignal, mode); return requestLookup( `${settings.apiBaseUrl}${RATE_API_V1_ENDPOINTS.EXTENSION_LOOKUP}?${params.toString()}` diff --git a/apps/extension/src/extractors/ecommerce-detector.ts b/apps/extension/src/extractors/ecommerce-detector.ts index 79803c325..c0a8f890e 100644 --- a/apps/extension/src/extractors/ecommerce-detector.ts +++ b/apps/extension/src/extractors/ecommerce-detector.ts @@ -475,27 +475,33 @@ export function extractPriceFromJsonLdOffers(offersRaw: unknown): { } /** - * A schema.org availability (`https://schema.org/InStock`, `InStock`, or a - * page's own "in stock" text) as a stock state; null when it says nothing - * (DOS/Rate#1606). Pre-orders and back-orders are unknown, not in stock. + * A schema.org availability (`https://schema.org/InStock`, `schema:InStock`, + * `InStock`, or a page's own "in stock" text) as a stock state; null when it + * says nothing (DOS/Rate#1606). Pre-orders and back-orders are unknown, not + * in stock. An array gives its first string. */ export function stockStateFromAvailability( raw: unknown ): ExtractedStockState | null { - if (typeof raw !== 'string' || raw.length > 200) return null; - const v = raw + const first = Array.isArray(raw) + ? raw.find((v): v is string => typeof v === 'string') + : raw; + if (typeof first !== 'string' || first.length > 200) return null; + const v = first .toLowerCase() - .replace(/^https?:\/\/schema\.org\//, '') + .replace(/^(?:https?:\/\/(?:www\.)?schema\.org\/|schema:)/, '') .replace(/[\s_-]+/g, ''); if (!v) return null; if (/^(?:instock|instoreonly|onlineonly|available)$/.test(v)) { return 'in_stock'; } if (/^(?:limitedavailability|lowstock)$/.test(v)) return 'low_stock'; - if (/^(?:outofstock|soldout|discontinued|oos)$/.test(v)) { + if (/^(?:outofstock|soldout|discontinued|reserved|oos)$/.test(v)) { return 'out_of_stock'; } - if (/^(?:preorder|presale|backorder|madetoorder)$/.test(v)) { + if ( + /^(?:preorder|presale|backorder|madetoorder|availablefororder)$/.test(v) + ) { return 'unknown'; } return null; @@ -515,11 +521,8 @@ function stockQtyOf(raw: unknown): number | null { return Number.isInteger(n) && n >= 0 && n <= 1_000_000 ? n : null; } -/** Stock state and quantity from JSON-LD offers: the first offer that says. */ -export function extractStockFromJsonLdOffers(offersRaw: unknown): { - stockState: ExtractedStockState | null; - stockQty: number | null; -} { +/** The offers of a JSON-LD `offers` value, nested AggregateOffer ones too. */ +export function jsonLdOfferList(offersRaw: unknown): Record[] { const offers: Record[] = []; const push = (entry: unknown) => { if ( @@ -532,29 +535,78 @@ export function extractStockFromJsonLdOffers(offersRaw: unknown): { } }; for (const entry of Array.isArray(offersRaw) ? offersRaw : [offersRaw]) { - push(entry); const nested = (entry as { offers?: unknown } | null)?.offers; - if (Array.isArray(nested)) nested.forEach(push); - } - for (const offer of offers) { - const stockState = stockStateFromAvailability(offer.availability); - if (stockState) { - return { - stockState, - stockQty: - stockState === 'out_of_stock' ? 0 : stockQtyOf(offer.inventoryLevel), - }; + if (nested && typeof nested === 'object') { + (Array.isArray(nested) ? nested : [nested]).forEach(push); + if (!(entry as { availability?: unknown }).availability) continue; } + push(entry); + } + return offers; +} + +/** + * The stock of one offer. A quantity is kept only when it agrees with the + * state: none for an unknown state, 0 for out of stock, and none for an + * "in stock" offer that says 0. + */ +export function stockOfOffer(offer: Record | null): { + stockState: ExtractedStockState | null; + stockQty: number | null; +} { + const stockState = offer + ? stockStateFromAvailability(offer.availability) + : null; + if (!stockState || stockState === 'unknown') { + return { stockState, stockQty: null }; + } + if (stockState === 'out_of_stock') return { stockState, stockQty: 0 }; + const qty = stockQtyOf(offer?.inventoryLevel); + return { stockState, stockQty: qty === 0 ? null : qty }; +} + +/** Stock state and quantity from JSON-LD offers: the first offer that says. */ +export function extractStockFromJsonLdOffers(offersRaw: unknown): { + stockState: ExtractedStockState | null; + stockQty: number | null; +} { + for (const offer of jsonLdOfferList(offersRaw)) { + const stock = stockOfOffer(offer); + if (stock.stockState) return stock; } return { stockState: null, stockQty: null }; } -const SKU_ID_RE = /^[A-Za-z0-9_.-]{1,64}$/; +/** + * The offer of the selected variant: the only offer, or the one whose sku or + * name matches the selected variant; null when several offers and none can be + * told apart, so stock and SKU are not read off another variant. + */ +export function pickVariantOffer( + offers: readonly Record[], + selectedVariant: string | null +): Record | null { + if (offers.length === 1) return offers[0] ?? null; + if (!selectedVariant) return null; + const wanted = selectedVariant.toLowerCase(); + const matches = offers.filter((offer) => + [offer.sku, offer.name].some( + (v) => + typeof v === 'string' && + v.trim().length > 0 && + wanted.includes(v.trim().toLowerCase()) + ) + ); + return matches.length === 1 ? (matches[0] ?? null) : null; +} + +const SKU_ID_RE = /^[A-Za-z0-9_-][A-Za-z0-9_.-]{0,63}$/; /** * The id of the selected variant: from the URL where the store puts it there - * (Lazada `-s.html`, Tiki `?spid=`), else the JSON-LD `sku` (DOS/Rate#1606). - * `rawUrl` is the page URL before cleaning, since Tiki keeps it in the query. + * (Lazada `-s.html`, Tiki `?spid=`), else the given structured-data sku + * (DOS/Rate#1606). `rawUrl` is the page URL before cleaning, since Tiki keeps + * it in the query. */ export function extractSkuId( platform: SupportedEcommercePlatform, @@ -750,11 +802,11 @@ export function extractProductSignalFromDom( let originalPrice: number | null = null; let ratingStar: number | null = null; let ratingCount: number | null = null; - let stockState: ExtractedStockState | null = null; - let stockQty: number | null = null; - let jsonLdSku: unknown = null; - let titleFromJsonLd = false; + // The first JSON-LD product's offers and sku (DOS/Rate#1606). + let productOffers: Record[] | null = null; + let productSku: unknown = null; let priceFromJsonLd = false; + let originalFromDom = false; // 1. Try JSON-LD Product structured data first (cleanest & resilient across SPA updates) const ldScripts = doc.querySelectorAll('script[type="application/ld+json"]'); @@ -766,18 +818,10 @@ export function extractProductSignalFromDom( if (isJsonLdProductEntity(item)) { if (!title && typeof item.name === 'string') { title = cleanExtractedProductTitle(item.name); - titleFromJsonLd = Boolean(title); } - if (!stockState) { - const stock = extractStockFromJsonLdOffers(item.offers); - stockState = stock.stockState; - stockQty = stock.stockQty; - } - if (jsonLdSku === null) { - const offerSku = Array.isArray(item.offers) - ? (item.offers[0] as { sku?: unknown } | undefined)?.sku - : (item.offers as { sku?: unknown } | undefined)?.sku; - jsonLdSku = item.sku ?? offerSku ?? null; + if (productOffers === null) { + productOffers = jsonLdOfferList(item.offers); + productSku = item.sku ?? null; } if (!price || !originalPrice) { const offerPrices = extractPriceFromJsonLdOffers(item.offers); @@ -854,24 +898,36 @@ export function extractProductSignalFromDom( const parsed = parseVndPriceText(el?.textContent); if (parsed && (!price || parsed > price)) { originalPrice = parsed; + originalFromDom = true; break; } } const selectedVariant = extractSelectedVariantFromDom(doc); - // 4. Stock from the page's product meta when JSON-LD said nothing. - if (!stockState) { + // 4. Stock and SKU from the selected variant's offer only, so they never + // describe another variant; then the page's product meta for the stock. + const offers = productOffers ?? []; + const offer = pickVariantOffer(offers, selectedVariant); + let { stockState, stockQty } = stockOfOffer(offer); + const stockFromJsonLd = stockState !== null; + if (!stockState && offers.length <= 1) { const meta = doc.querySelector( - 'meta[property="product:availability"], meta[property="og:availability"], [itemprop="availability"]' - ); - stockState = stockStateFromAvailability( - meta?.getAttribute('content') ?? meta?.getAttribute('href') + 'meta[property="product:availability"], meta[property="og:availability"]' ); - if (stockState === 'out_of_stock') stockQty = 0; - } - - const fromJsonLd = [titleFromJsonLd, priceFromJsonLd || price === null]; + ({ stockState, stockQty } = stockOfOffer({ + availability: meta?.getAttribute('content'), + })); + } + const jsonLdSku = + offer?.sku ?? (offers.length <= 1 ? productSku : null) ?? null; + + // Where the fields v1 carries came from: price, list price and stock. + const sources = [ + price !== null ? priceFromJsonLd : null, + originalPrice !== null ? !originalFromDom : null, + stockState !== null ? stockFromJsonLd : null, + ].filter((v): v is boolean => v !== null); return { platform, canonicalUrl, @@ -886,10 +942,11 @@ export function extractProductSignalFromDom( skuId: extractSkuId(platform, currentUrl, jsonLdSku), stockState: stockState ?? 'unknown', stockQty: stockState ? stockQty : null, - extractorSource: fromJsonLd.every(Boolean) - ? 'json_ld' - : fromJsonLd.some(Boolean) - ? 'mixed' - : 'dom', + extractorSource: + sources.length > 0 && sources.every(Boolean) + ? 'json_ld' + : sources.some(Boolean) + ? 'mixed' + : 'dom', }; } diff --git a/apps/extension/src/security-guards.ts b/apps/extension/src/security-guards.ts index e5ebdc02a..8c2878ace 100644 --- a/apps/extension/src/security-guards.ts +++ b/apps/extension/src/security-guards.ts @@ -291,17 +291,20 @@ export function sanitizeExtensionProductSignal( ratingStar: validRatingStar, ratingCount: validRatingCount, // The v1 observation fields (DOS/Rate#1606), checked like the rest. - skuId: - typeof safeSignal.skuId === 'string' && - /^[A-Za-z0-9_.-]{1,64}$/.test(safeSignal.skuId) - ? safeSignal.skuId - : null, + skuId: (() => { + const sku = sanitizeExternalIdentifier(safeSignal.skuId); + return sku && sku.length <= 64 ? sku : null; + })(), stockState: ( ['in_stock', 'low_stock', 'out_of_stock', 'unknown'] as const ).includes(safeSignal.stockState as never) ? safeSignal.stockState : 'unknown', + // A quantity only with a known state (DOS/Rate#1606). stockQty: + ['in_stock', 'low_stock', 'out_of_stock'].includes( + safeSignal.stockState as string + ) && typeof safeSignal.stockQty === 'number' && Number.isInteger(safeSignal.stockQty) && safeSignal.stockQty >= 0 && @@ -316,6 +319,35 @@ export function sanitizeExtensionProductSignal( }; } +/** + * The query of the product lookup: exactly the fields the disclosures name + * (store, cleaned URL, title, item and shop ids, variant, price, original + * price) and the mode. A test pins its output (DOS/Rate#1606). + */ +export function buildLookupParams( + rawSignal: ExtractedPageProductSignal, + mode: 'pill' | 'full' +): URLSearchParams { + const signal = sanitizeExtensionProductSignal(rawSignal); + const params = new URLSearchParams(); + params.set('platform', signal.platform); + params.set('url', signal.canonicalUrl); + params.set('title', signal.title); + params.set('mode', mode); + if (signal.externalItemId) params.set('externalId', signal.externalItemId); + if (signal.externalShopId) params.set('shopId', signal.externalShopId); + if (signal.selectedVariant) { + params.set('selectedVariant', signal.selectedVariant); + } + if (signal.price && signal.price > 0) { + params.set('price', String(signal.price)); + } + if (signal.originalPrice && signal.originalPrice > 0) { + params.set('originalPrice', String(signal.originalPrice)); + } + return params; +} + /** * The ExtensionObservation v1 an opted-in install would send for `signal` * (DOS/Rate#1606), or null when the page lacks what v1 needs (an item id, a @@ -336,7 +368,12 @@ export function buildExtensionObservationV1( external_item_id: signal.externalItemId, external_shop_id: signal.externalShopId, sku_id: signal.skuId ?? null, - variant_label: signal.selectedVariant, + // v1 allows 120 characters; a longer label is cut, not a reason to + // drop the observation. + variant_label: signal.selectedVariant + ? Array.from(signal.selectedVariant).slice(0, 120).join('').trim() || + null + : null, price_vnd: signal.price, list_price_vnd: signal.originalPrice, stock_state: signal.stockState ?? 'unknown', diff --git a/apps/extension/src/test-extension-pipeline.ts b/apps/extension/src/test-extension-pipeline.ts index 40aada62e..748237425 100644 --- a/apps/extension/src/test-extension-pipeline.ts +++ b/apps/extension/src/test-extension-pipeline.ts @@ -4995,11 +4995,18 @@ async function runTests() { path.resolve(__dirname, 'background/service-worker.ts'), 'utf8' ); + // The lookup query is built in security-guards (DOS/Rate#1606). + const guardsSource = fs.readFileSync( + path.resolve(__dirname, 'security-guards.ts'), + 'utf8' + ); const sentParams = [ ...new Set( - [...workerSource.matchAll(/params\.set\(\s*'([^']+)'/g)].map( - (m) => m[1]! - ) + [ + ...(workerSource + guardsSource).matchAll( + /params\.set\(\s*'([^']+)'/g + ), + ].map((m) => m[1]!) ), ].sort(); assert.deepEqual(sentParams, [ @@ -5167,11 +5174,139 @@ async function runTests() { for (const key of ['skuId', 'stockState', 'stockQty', 'extractorSource']) { assert.ok(!(key in crowd), key); } + // The exact lookup query and opt-in payload, pinned: they are what the + // disclosures describe. + assert.equal( + guards.buildLookupParams(signal, 'pill').toString(), + 'platform=shopee&url=https%3A%2F%2Fshopee.vn%2Fiphone-16-i.123.456&title=iPhone+16+128GB&mode=pill&externalId=456&shopId=123&selectedVariant=%C4%90en&price=19990000&originalPrice=22990000' + ); + assert.equal( + JSON.stringify(crowd), + JSON.stringify({ + platform: 'shopee', + externalItemId: '456', + externalShopId: '123', + url: 'https://shopee.vn/iphone-16-i.123.456', + title: 'iPhone 16 128GB', + selectedVariant: 'Đen', + price: 19_990_000, + originalPrice: 22_990_000, + ratingStar: 4.9, + ratingCount: 1000, + userConsentOptIn: true, + }) + ); const workerSource = fs.readFileSync( path.resolve(__dirname, 'background/service-worker.ts'), 'utf8' ); - assert.ok(!/stockState|skuId|stockQty/.test(workerSource)); + assert.ok(!/stockState|skuId|stockQty|extractorSource/.test(workerSource)); + assert.ok(workerSource.includes('buildLookupParams(rawSignal, mode)')); + + // A long variant label is cut to 120 characters, not a lost observation. + const longObs = guards.buildExtensionObservationV1( + { ...signal, selectedVariant: 'Màu '.repeat(50) }, + at, + '2.1.0' + ); + assert.ok(longObs && longObs.variant_label!.length <= 120); + + // More availability forms; quantities only agree with the state. + assert.equal(det.stockStateFromAvailability('schema:InStock'), 'in_stock'); + assert.equal( + det.stockStateFromAvailability('https://www.schema.org/SoldOut'), + 'out_of_stock' + ); + assert.equal(det.stockStateFromAvailability('Reserved'), 'out_of_stock'); + assert.equal(det.stockStateFromAvailability(['InStock']), 'in_stock'); + assert.equal( + det.stockStateFromAvailability('available for order'), + 'unknown' + ); + assert.deepEqual( + det.stockOfOffer({ availability: 'InStock', inventoryLevel: 0 }), + { + stockState: 'in_stock', + stockQty: null, + } + ); + assert.deepEqual( + det.stockOfOffer({ availability: 'PreOrder', inventoryLevel: 50 }), + { + stockState: 'unknown', + stockQty: null, + } + ); + assert.deepEqual( + det.extractStockFromJsonLdOffers({ + '@type': 'AggregateOffer', + offers: { availability: 'LimitedAvailability', inventoryLevel: '3' }, + }), + { stockState: 'low_stock', stockQty: 3 } + ); + assert.equal( + det.extractSkuId('shopee', 'https://shopee.vn/x-i.1.2', '.'), + null + ); + + // End to end on a page: the selected variant's own offer gives stock and + // SKU, never another variant's or the parent's. + const variantPage = (selected: string | null, offers: unknown) => + ({ + title: 'iPhone 16', + querySelectorAll(sel: string) { + if (sel === 'script[type="application/ld+json"]') { + return [ + { + textContent: JSON.stringify({ + '@type': 'Product', + name: 'iPhone 16 128GB', + sku: 'BASE-1', + offers, + }), + }, + ]; + } + if (selected && sel.includes('aria-checked')) { + return [{ textContent: selected }]; + } + return []; + }, + querySelector() { + return null; + }, + }) as unknown as Document; + const twoOffers = [ + { sku: 'V-RED', name: 'Đỏ', price: 19990000, availability: 'OutOfStock' }, + { + sku: 'V-BLUE', + name: 'Xanh', + price: 19990000, + availability: 'InStock', + inventoryLevel: 4, + }, + ]; + const url = 'https://shopee.vn/iphone-16-i.123.456'; + const none = det.extractProductSignalFromDom( + variantPage(null, twoOffers), + url + ); + assert.ok(none); + assert.equal(none.stockState, 'unknown'); + assert.equal(none.skuId, null); + assert.equal(none.extractorSource, 'json_ld'); + const single = det.extractProductSignalFromDom( + variantPage(null, { + price: 19990000, + availability: 'InStock', + inventoryLevel: 9, + }), + url + ); + assert.ok(single); + assert.equal(single.stockState, 'in_stock'); + assert.equal(single.stockQty, 9); + assert.equal(single.skuId, 'BASE-1'); console.log( ' ✅ [Stock, SKU and the v1 observation, not sent (#1606)] Verified' ); diff --git a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts index 857419735..c4e040f81 100644 --- a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts +++ b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts @@ -878,22 +878,76 @@ async function testExtensionObservations() { } assert.equal(calls.length, 0, 'a disabled route touches nothing'); + // A preflight is a 404 too while off. + assert.equal( + observationRoute.OPTIONS( + new Request('http://localhost/api/v1/rate/extension/observation', { + method: 'OPTIONS', + }) + ).status, + 404 + ); + process.env.RATE_EXTENSION_OBSERVATIONS = '1'; const savedSalt = process.env.RATE_NETWORK_HASH_SALT; + const savedKey = process.env.RATE_EXTENSION_INGEST_KEY; + const INGEST_KEY = 'ingest-key-0123456789abcdef0123456789'; try { - // On, but without the network hash salt: observations are refused, - // since the per-network limit would not hold. - delete process.env.RATE_NETWORK_HASH_SALT; + // On, but without the salt or the ingest key: writes are refused before + // the database, since its limits and key check would not hold. + for (const [salt, key] of [ + [undefined, INGEST_KEY], + ['test-salt-0123456789', undefined], + ['test-salt-0123456789', 'too-short'], + ] as const) { + if (salt) process.env.RATE_NETWORK_HASH_SALT = salt; + else delete process.env.RATE_NETWORK_HASH_SALT; + if (key) process.env.RATE_EXTENSION_INGEST_KEY = key; + else delete process.env.RATE_EXTENSION_INGEST_KEY; + calls = []; + for (const res of [ + await submitterRoute.POST(post('/api/v1/rate/extension/submitter', {})), + await observationRoute.POST( + post('/api/v1/rate/extension/observation', { + submitterId: SUBMITTER, + observation, + }) + ), + ]) { + assert.equal(res.status, 503); + } + assert.equal(calls.length, 0); + } + process.env.RATE_NETWORK_HASH_SALT = 'test-salt-0123456789'; + process.env.RATE_EXTENSION_INGEST_KEY = INGEST_KEY; + + // A web page cannot call the routes; an extension origin gets CORS; a + // body that is not JSON is refused. calls = []; - const noSalt = await observationRoute.POST( - post('/api/v1/rate/extension/observation', { - submitterId: SUBMITTER, - observation, - }) + const fromSite = post('/api/v1/rate/extension/submitter', {}); + fromSite.headers.set('origin', 'https://evil.example'); + assert.equal((await submitterRoute.POST(fromSite)).status, 403); + const asText = new Request( + 'http://localhost/api/v1/rate/extension/observation', + { + method: 'POST', + headers: { 'Content-Type': 'text/plain' }, + body: JSON.stringify({ submitterId: SUBMITTER, observation }), + } ); - assert.equal(noSalt.status, 503); + assert.equal((await observationRoute.POST(asText)).status, 415); assert.equal(calls.length, 0); - process.env.RATE_NETWORK_HASH_SALT = 'test-salt-0123456789'; + const preflight = observationRoute.OPTIONS( + new Request('http://localhost/api/v1/rate/extension/observation', { + method: 'OPTIONS', + headers: { origin: 'chrome-extension://abcdefghijklmnop' }, + }) + ); + assert.equal(preflight.status, 204); + assert.equal( + preflight.headers.get('access-control-allow-origin'), + 'chrome-extension://abcdefghijklmnop' + ); // A submitter id comes from the RPC, as anon, never the service key. calls = []; @@ -906,6 +960,11 @@ async function testExtensionObservations() { ); assert.equal(res.status, 200); assert.deepEqual(await res.json(), { ok: true, submitterId: SUBMITTER }); + assert.deepEqual(rpcCalls('register_extension_submitter')[0]?.body, { + p_ingest_key: INGEST_KEY, + p_network_hash: obsMod.networkHashFor('anon', 'test-salt-0123456789'), + }); + assert.equal(res.headers.get('access-control-allow-origin'), null); assert.match(res.headers.get('cache-control') ?? '', /no-store/); assert.ok( calls.every((c) => !JSON.stringify(c.headers).includes(SERVICE_KEY)) @@ -934,6 +993,7 @@ async function testExtensionObservations() { const [call] = rpcCalls('submit_extension_observation'); assert.ok(call, 'submit_extension_observation must be called'); const args = call.body as Record; + assert.equal(args.p_ingest_key, INGEST_KEY); assert.equal(args.p_submitter_id, SUBMITTER); assert.deepEqual(args.p_observation, observation); assert.equal( @@ -982,6 +1042,11 @@ async function testExtensionObservations() { [{ code: '22023', details: 'price_vnd' }, 400, 'INVALID_OBSERVATION'], [{ code: 'P0002', details: 'submitter_id' }, 404, 'UNKNOWN_SUBMITTER'], [{ code: '54000' }, 429, 'RATE_LIMIT_EXCEEDED'], + [ + { code: '42501', details: 'ingest_key' }, + 503, + 'OBSERVATIONS_UNAVAILABLE', + ], [ { code: 'PGRST202', message: 'Could not find' }, 503, @@ -1003,6 +1068,9 @@ async function testExtensionObservations() { const json = await res.json(); assert.equal(json.code, code, error.code); if (error.code === '22023') assert.equal(json.field, 'price_vnd'); + if (error.code === '54000') { + assert.equal(res.headers.get('retry-after'), '3600'); + } } // Deleting answers the same for any id and calls the delete RPC. @@ -1031,6 +1099,10 @@ async function testExtensionObservations() { '2001:db8:0::/48' ); assert.equal(obsMod.networkPrefixOf('::1'), '0:0:0::/48'); + assert.equal( + obsMod.networkPrefixOf('::ffff:203.0.113.9'), + '203.0.113.0/24' + ); for (const bad of [ 'anon', '999.1.1.1', @@ -1042,7 +1114,15 @@ async function testExtensionObservations() { assert.equal(obsMod.networkPrefixOf(bad), null, bad); } assert.equal(obsMod.networkHashFor('203.0.113.77', 'short'), null); - assert.equal(obsMod.networkHashFor('anon', 'test-salt-0123456789'), null); + // An unusable address still lands in one shared bucket. + assert.match( + String(obsMod.networkHashFor('anon', 'test-salt-0123456789')), + /^[0-9a-f]{64}$/ + ); + assert.equal( + obsMod.networkHashFor('anon', 'test-salt-0123456789'), + obsMod.networkHashFor('1.2.3', 'test-salt-0123456789') + ); assert.equal( obsMod.networkHashFor('203.0.113.5', 'test-salt-0123456789'), obsMod.networkHashFor('203.0.113.200', 'test-salt-0123456789') @@ -1055,6 +1135,8 @@ async function testExtensionObservations() { delete process.env.RATE_EXTENSION_OBSERVATIONS; if (savedSalt === undefined) delete process.env.RATE_NETWORK_HASH_SALT; else process.env.RATE_NETWORK_HASH_SALT = savedSalt; + if (savedKey === undefined) delete process.env.RATE_EXTENSION_INGEST_KEY; + else process.env.RATE_EXTENSION_INGEST_KEY = savedKey; } } diff --git a/apps/web/src/app/api/v1/rate/extension/observation/route.ts b/apps/web/src/app/api/v1/rate/extension/observation/route.ts index 6157ccfec..17bafadfa 100644 --- a/apps/web/src/app/api/v1/rate/extension/observation/route.ts +++ b/apps/web/src/app/api/v1/rate/extension/observation/route.ts @@ -7,15 +7,22 @@ import { readBoundedJsonBody, } from '@/server/rate/api-handler'; import { - EXTENSION_OBSERVATION_HEADERS, callObservationRpc, hasNetworkHashSalt, + ingestKey, isExtensionObservationsEnabled, + isJsonRequest, + isWebPageOrigin, networkHashFor, notEnabledResponse, + notJsonResponse, + observationHeaders, + observationOptions, observationResponse, parseSubmitterId, rpcFailureResponse, + unavailableResponse, + webPageRefusedResponse, } from '@/server/rate/extension-observations'; import { createLogger, serializeError } from '@/lib/logger'; @@ -29,11 +36,8 @@ const observationLimiter = new BoundedSlidingWindowRateLimiter({ maxBuckets: 2_000, }); -export async function OPTIONS() { - return new Response(null, { - status: 204, - headers: EXTENSION_OBSERVATION_HEADERS, - }); +export function OPTIONS(request: Request) { + return observationOptions(request); } /** @@ -43,31 +47,28 @@ export async function OPTIONS() { * on: only a salted hash of its network range goes to the RPC. */ export async function POST(request: Request) { - if (!isExtensionObservationsEnabled()) return notEnabledResponse(); - if (!hasNetworkHashSalt()) { - return observationResponse(503, { - ok: false, - code: 'OBSERVATIONS_UNAVAILABLE', - error: 'Extension observations are not available yet.', - }); - } + if (!isExtensionObservationsEnabled()) return notEnabledResponse(request); + if (isWebPageOrigin(request)) return webPageRefusedResponse(request); + if (!isJsonRequest(request)) return notJsonResponse(request); + const key = ingestKey(); + if (!key || !hasNetworkHashSalt()) return unavailableResponse(request); const clientIp = extractClientRateLimitKey(request.headers); if (!observationLimiter.allow(clientIp)) { return rateLimitExceededResponse( 'Too many observations; try again in a minute.', - EXTENSION_OBSERVATION_HEADERS + observationHeaders(request) ); } const body = await readBoundedJsonBody(request, 8 * 1024); if (body.tooLarge) - return payloadTooLargeResponse(EXTENSION_OBSERVATION_HEADERS); + return payloadTooLargeResponse(observationHeaders(request)); const raw = body.value && typeof body.value === 'object' && !Array.isArray(body.value) ? (body.value as { submitterId?: unknown; observation?: unknown }) : {}; const submitterId = parseSubmitterId(raw.submitterId); if (!submitterId) { - return observationResponse(400, { + return observationResponse(request, 400, { ok: false, code: 'INVALID_SUBMITTER', error: 'Provide submitterId as a uuid.', @@ -75,7 +76,7 @@ export async function POST(request: Request) { } const parsed = parseExtensionObservationV1(raw.observation); if (!parsed.ok) { - return observationResponse(400, { + return observationResponse(request, 400, { ok: false, code: 'INVALID_OBSERVATION', error: 'The observation was refused.', @@ -86,17 +87,18 @@ export async function POST(request: Request) { const { data, error } = await callObservationRpc( 'submit_extension_observation', { + p_ingest_key: key, p_submitter_id: submitterId, p_observation: parsed.value, p_network_hash: networkHashFor(clientIp), } ); - if (error) return rpcFailureResponse(error); + if (error) return rpcFailureResponse(request, error); const stored = data && typeof data === 'object' && !Array.isArray(data) ? (data as { status?: unknown }) : null; - return observationResponse(200, { + return observationResponse(request, 200, { ok: true, status: stored?.status === 'quarantined' ? 'quarantined' : 'received', }); @@ -104,6 +106,6 @@ export async function POST(request: Request) { log.error('submit_extension_observation failed', { error: serializeError(err), }); - return rpcFailureResponse({}); + return rpcFailureResponse(request, {}); } } diff --git a/apps/web/src/app/api/v1/rate/extension/submitter/route.ts b/apps/web/src/app/api/v1/rate/extension/submitter/route.ts index 35f18738f..1c3ccb2ed 100644 --- a/apps/web/src/app/api/v1/rate/extension/submitter/route.ts +++ b/apps/web/src/app/api/v1/rate/extension/submitter/route.ts @@ -6,19 +6,28 @@ import { readBoundedJsonBody, } from '@/server/rate/api-handler'; import { - EXTENSION_OBSERVATION_HEADERS, callObservationRpc, + hasNetworkHashSalt, + ingestKey, isExtensionObservationsEnabled, + isJsonRequest, + isWebPageOrigin, + networkHashFor, notEnabledResponse, + notJsonResponse, + observationHeaders, + observationOptions, observationResponse, parseSubmitterId, rpcFailureResponse, + unavailableResponse, + webPageRefusedResponse, } from '@/server/rate/extension-observations'; import { createLogger, serializeError } from '@/lib/logger'; const log = createLogger('api.extension.submitter'); -// Best-effort per-instance brakes; the RPCs hold the global ones. +// Best-effort per-instance brakes; the RPCs hold the real limits. const registerLimiter = new BoundedSlidingWindowRateLimiter({ windowMs: 60_000, maxRequests: 10, @@ -30,45 +39,47 @@ const deleteLimiter = new BoundedSlidingWindowRateLimiter({ maxBuckets: 2_000, }); -export async function OPTIONS() { - return new Response(null, { - status: 204, - headers: EXTENSION_OBSERVATION_HEADERS, - }); +export function OPTIONS(request: Request) { + return observationOptions(request); } /** * POST /api/v1/rate/extension/submitter: a new pseudonymous submitter id for * an install that opted in (DOS/Rate#1606). No body; the server creates the - * id and records when. + * id and records when, and the RPC limits new ids per network. */ export async function POST(request: Request) { - if (!isExtensionObservationsEnabled()) return notEnabledResponse(); - if (!registerLimiter.allow(extractClientRateLimitKey(request.headers))) { + if (!isExtensionObservationsEnabled()) return notEnabledResponse(request); + if (isWebPageOrigin(request)) return webPageRefusedResponse(request); + const key = ingestKey(); + if (!key || !hasNetworkHashSalt()) return unavailableResponse(request); + const clientIp = extractClientRateLimitKey(request.headers); + if (!registerLimiter.allow(clientIp)) { return rateLimitExceededResponse( 'Too many requests; try again in a minute.', - EXTENSION_OBSERVATION_HEADERS + observationHeaders(request) ); } try { const { data, error } = await callObservationRpc( - 'register_extension_submitter' + 'register_extension_submitter', + { p_ingest_key: key, p_network_hash: networkHashFor(clientIp) } ); - if (error) return rpcFailureResponse(error); + if (error) return rpcFailureResponse(request, error); const submitterId = parseSubmitterId(data); if (!submitterId) { - return observationResponse(502, { + return observationResponse(request, 502, { ok: false, code: 'PERSIST_FAILED', error: 'Could not create a submitter id.', }); } - return observationResponse(200, { ok: true, submitterId }); + return observationResponse(request, 200, { ok: true, submitterId }); } catch (err) { log.error('register_extension_submitter failed', { error: serializeError(err), }); - return rpcFailureResponse({}); + return rpcFailureResponse(request, {}); } } @@ -78,21 +89,23 @@ export async function POST(request: Request) { * erasure). The answer is the same whether or not the id existed. */ export async function DELETE(request: Request) { - if (!isExtensionObservationsEnabled()) return notEnabledResponse(); + if (!isExtensionObservationsEnabled()) return notEnabledResponse(request); + if (isWebPageOrigin(request)) return webPageRefusedResponse(request); + if (!isJsonRequest(request)) return notJsonResponse(request); if (!deleteLimiter.allow(extractClientRateLimitKey(request.headers))) { return rateLimitExceededResponse( 'Too many requests; try again in a minute.', - EXTENSION_OBSERVATION_HEADERS + observationHeaders(request) ); } const body = await readBoundedJsonBody(request, 1024); if (body.tooLarge) - return payloadTooLargeResponse(EXTENSION_OBSERVATION_HEADERS); + return payloadTooLargeResponse(observationHeaders(request)); const submitterId = parseSubmitterId( (body.value as { submitterId?: unknown } | null)?.submitterId ); if (!submitterId) { - return observationResponse(400, { + return observationResponse(request, 400, { ok: false, code: 'INVALID_SUBMITTER', error: 'Provide submitterId as a uuid.', @@ -102,12 +115,12 @@ export async function DELETE(request: Request) { const { error } = await callObservationRpc('delete_extension_submitter', { p_submitter_id: submitterId, }); - if (error) return rpcFailureResponse(error); - return observationResponse(200, { ok: true }); + if (error) return rpcFailureResponse(request, error); + return observationResponse(request, 200, { ok: true }); } catch (err) { log.error('delete_extension_submitter failed', { error: serializeError(err), }); - return rpcFailureResponse({}); + return rpcFailureResponse(request, {}); } } diff --git a/apps/web/src/server/rate/extension-observations.ts b/apps/web/src/server/rate/extension-observations.ts index 61252da5f..9a81622e8 100644 --- a/apps/web/src/server/rate/extension-observations.ts +++ b/apps/web/src/server/rate/extension-observations.ts @@ -3,6 +3,9 @@ import { NextResponse } from 'next/server'; import { getRateClient } from '@/lib/rate-api'; import { isMissingRpcError } from '@/server/rate/request-auth'; import { rateLimitExceededResponse } from '@/server/rate/api-handler'; +import { createLogger } from '@/lib/logger'; + +const log = createLogger('server.extension-observations'); /** * ExtensionObservation v1 ingest (DOS/Rate#1606, M2 of @@ -14,23 +17,57 @@ import { rateLimitExceededResponse } from '@/server/rate/api-handler'; * * Off until the table exists: every route answers 404 unless * RATE_EXTENSION_OBSERVATIONS is "1", and then makes no database call. + * Writes also need two server secrets: RATE_EXTENSION_INGEST_KEY, which the + * RPCs check so nobody can call them around this route, and + * RATE_NETWORK_HASH_SALT for the per-network limits. */ -export const EXTENSION_OBSERVATION_HEADERS: Record = { - 'Access-Control-Allow-Origin': '*', +const BASE_HEADERS: Record = { 'Access-Control-Allow-Methods': 'POST, DELETE, OPTIONS', 'Access-Control-Allow-Headers': 'Content-Type, X-Rate-Extension-Version', 'Cache-Control': 'no-store', + Vary: 'Origin', }; +/** Browser-extension origins: the only browser callers these routes serve. */ +const EXTENSION_ORIGIN_RE = + /^(?:chrome-extension|moz-extension|extension):\/\/[a-z0-9-]{1,64}$/i; + +/** + * The response headers for a request: CORS only for an extension origin. + * Callers without an Origin (the extension's service worker sends one; + * server tools send none) get no CORS header at all. + */ +export function observationHeaders( + request?: Request | null +): Record { + const origin = request?.headers.get('origin') ?? ''; + return EXTENSION_ORIGIN_RE.test(origin) + ? { ...BASE_HEADERS, 'Access-Control-Allow-Origin': origin } + : { ...BASE_HEADERS }; +} + +/** + * A web page's request: any http(s) Origin. Refused, so a site the user + * visits cannot register ids or send observations from the user's network. + */ +export function isWebPageOrigin(request: Request): boolean { + return /^https?:\/\//i.test(request.headers.get('origin') ?? ''); +} + +export function isJsonRequest(request: Request): boolean { + return /^application\/json\b/i.test( + request.headers.get('content-type') ?? '' + ); +} + export function isExtensionObservationsEnabled(): boolean { return process.env.RATE_EXTENSION_OBSERVATIONS === '1'; } /** - * The per-network limit needs the salt: without it every request's - * network_hash is null and fresh submitter ids walk around the - * per-submitter limit. Observations are refused until it is set. + * The per-network limits need the salt, and the RPCs need the ingest key; + * writes are refused until both are set. */ export function hasNetworkHashSalt( salt = process.env.RATE_NETWORK_HASH_SALT @@ -38,23 +75,51 @@ export function hasNetworkHashSalt( return typeof salt === 'string' && salt.length >= 16; } +export function ingestKey( + key = process.env.RATE_EXTENSION_INGEST_KEY +): string | null { + return typeof key === 'string' && key.length >= 32 ? key : null; +} + export function observationResponse( + request: Request | null, status: number, body: Record ): NextResponse { return NextResponse.json(body, { status, - headers: EXTENSION_OBSERVATION_HEADERS, + headers: observationHeaders(request), }); } -export const notEnabledResponse = () => - observationResponse(404, { +export const notEnabledResponse = (request: Request | null) => + observationResponse(request, 404, { ok: false, code: 'NOT_ENABLED', error: 'Extension observations are not enabled.', }); +export const unavailableResponse = (request: Request | null) => + observationResponse(request, 503, { + ok: false, + code: 'OBSERVATIONS_UNAVAILABLE', + error: 'Extension observations are not available yet.', + }); + +export const webPageRefusedResponse = (request: Request | null) => + observationResponse(request, 403, { + ok: false, + code: 'CROSS_SITE', + error: 'Web pages cannot call this endpoint.', + }); + +export const notJsonResponse = (request: Request | null) => + observationResponse(request, 415, { + ok: false, + code: 'UNSUPPORTED_MEDIA_TYPE', + error: 'Send Content-Type: application/json.', + }); + const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; @@ -65,11 +130,15 @@ export function parseSubmitterId(raw: unknown): string | null { } /** - * The network range of an IP: the /24 of IPv4, the /48 of IPv6; null for - * anything else (no address, a placeholder, a malformed value). + * The network range of an IP: the /24 of IPv4 (also IPv4-mapped IPv6), the + * /48 of IPv6; null for anything else (no address, a placeholder, a + * malformed value). */ export function networkPrefixOf(ip: string): string | null { - const v4 = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(ip); + const mapped = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/i.exec(ip); + const v4 = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec( + mapped?.[1] ?? ip + ); if (v4) { const parts = v4.slice(1, 5).map(Number); if (parts.some((p) => p > 255)) return null; @@ -93,18 +162,19 @@ export function networkPrefixOf(ip: string): string | null { /** * HMAC-SHA256 of the network range with RATE_NETWORK_HASH_SALT, as lowercase * hex: lets Rate-Services tell independent submitters apart without the raw - * IP ever being stored. Null without a salt of at least 16 characters or - * without a usable address. + * IP ever being stored. An address that gives no range shares one "unknown" + * bucket, so the per-network limits still hold. Null only without a salt of + * at least 16 characters. */ export function networkHashFor( ip: string, salt = process.env.RATE_NETWORK_HASH_SALT ): string | null { - if (typeof salt !== 'string' || salt.length < 16) return null; - const prefix = networkPrefixOf(ip); - return prefix - ? createHmac('sha256', salt).update(prefix).digest('hex') - : null; + if (!hasNetworkHashSalt(salt)) return null; + const prefix = networkPrefixOf(ip) ?? 'unknown'; + return createHmac('sha256', salt as string) + .update(prefix) + .digest('hex'); } interface LooseRpcClient { @@ -128,47 +198,62 @@ export function callObservationRpc( return (getRateClient() as unknown as LooseRpcClient).rpc(fn, args); } -/** An RPC failure as an answer: never a success, never the raw message. */ -export function rpcFailureResponse(error: { - code?: string; - details?: string; -}): NextResponse { - if (isMissingRpcError(error)) { - return observationResponse(503, { - ok: false, - code: 'OBSERVATIONS_UNAVAILABLE', - error: 'Extension observations are not available yet.', - }); - } +/** + * An RPC failure as an answer: never a success, never the raw message. The + * code is logged (not the message, which can carry input). + */ +export function rpcFailureResponse( + request: Request | null, + error: { code?: string; details?: string } +): NextResponse { + log.warn('extension observation rpc refused', { + code: typeof error.code === 'string' ? error.code.slice(0, 16) : null, + }); + if (isMissingRpcError(error)) return unavailableResponse(request); if (error.code === '54000') { + // The database limits are hourly. return rateLimitExceededResponse( - 'Too many observations; try again later.', - EXTENSION_OBSERVATION_HEADERS + 'Too many requests; try again later.', + observationHeaders(request), + 3600 ); } if (error.code === 'P0002') { - return observationResponse(404, { + return observationResponse(request, 404, { ok: false, code: 'UNKNOWN_SUBMITTER', error: 'Unknown submitter.', }); } + if (error.code === '42501') { + // The ingest key was refused: a server configuration problem. + return unavailableResponse(request); + } if (error.code === '22023') { const field = typeof error.details === 'string' && /^[a-z_]{1,32}(?:\.[a-z_]{1,32})?$/.test(error.details) ? error.details : null; - return observationResponse(400, { + return observationResponse(request, 400, { ok: false, code: 'INVALID_OBSERVATION', error: 'The observation was refused.', field, }); } - return observationResponse(502, { + return observationResponse(request, 502, { ok: false, code: 'PERSIST_FAILED', error: 'Could not store the observation.', }); } + +/** The answer to a preflight: 404 while the routes are off. */ +export function observationOptions(request: Request): Response { + if (!isExtensionObservationsEnabled()) return notEnabledResponse(request); + return new Response(null, { + status: 204, + headers: observationHeaders(request), + }); +} diff --git a/packages/shared-data/src/__tests__/shared-data.test.ts b/packages/shared-data/src/__tests__/shared-data.test.ts index 9157f0345..5b6acb9a6 100644 --- a/packages/shared-data/src/__tests__/shared-data.test.ts +++ b/packages/shared-data/src/__tests__/shared-data.test.ts @@ -2377,6 +2377,29 @@ function testExtensionObservation(): void { { ...valid, listing_url: `https://shopee.vn/${'a'.repeat(2048)}` }, 'listing_url', ], + // The RPC's URL rules: lowercase host, no port, user, path parameter + // or space; no empty subdomain label. + [{ ...valid, listing_url: 'https://Shopee.vn/x' }, 'listing_url'], + [{ ...valid, listing_url: 'https://shopee.vn:443/x' }, 'listing_url'], + [{ ...valid, listing_url: 'https://@shopee.vn/x' }, 'listing_url'], + [ + { ...valid, listing_url: 'https://shopee.vn/x;jsessionid=1' }, + 'listing_url', + ], + [{ ...valid, listing_url: 'https://shopee.vn/a b' }, 'listing_url'], + [{ ...valid, listing_url: 'https:shopee.vn/x' }, 'listing_url'], + [ + { + ...valid, + marketplace: 'lazada', + listing_url: 'https://a..lazada.vn/x', + }, + 'listing_url', + ], + [{ ...valid, variant_label: 'a\ud800b' }, 'variant_label'], + [{ ...valid, observed_at: '2026-11-04' }, 'observed_at'], + [{ ...valid, observed_at: '2026-11-04T09:59' }, 'observed_at'], + [{ ...valid, observed_at: '2026-11-04 09:59Z' }, 'observed_at'], [{ ...valid, external_item_id: '' }, 'external_item_id'], [{ ...valid, external_item_id: 'a b' }, 'external_item_id'], [{ ...valid, external_shop_id: 'x'.repeat(65) }, 'external_shop_id'], @@ -2413,6 +2436,21 @@ function testExtensionObservation(): void { JSON.stringify(input).slice(0, 80) ); } + // Offsets the RPC accepts are accepted here too. + for (const at of [ + '2026-11-04T16:59+07', + '2026-11-04T16:59+0700', + '2026-11-04T16:59:00.123456+07:00', + ]) { + assert.ok(parse({ ...valid, observed_at: at }).ok, at); + } + assert.ok( + parse({ + ...valid, + marketplace: 'lazada', + listing_url: 'https://www.lazada.vn/products/x-i1-s2.html', + }).ok + ); // Every marketplace accepts its own hosts and refuses another's. assert.ok(rules.isExtensionMarketplaceHost('lazada', 'www.lazada.vn')); assert.ok(rules.isExtensionMarketplaceHost('tgdd', 'www.thegioididong.com')); diff --git a/packages/shared-data/src/rules/extension-observation.ts b/packages/shared-data/src/rules/extension-observation.ts index d0d112d21..9d65f70ea 100644 --- a/packages/shared-data/src/rules/extension-observation.ts +++ b/packages/shared-data/src/rules/extension-observation.ts @@ -148,37 +148,62 @@ export function isExtensionMarketplaceHost( ): boolean { const h = host.toLowerCase(); return EXTENSION_MARKETPLACE_HOSTS[marketplace].some((allowed) => - allowed.startsWith('.') ? h.endsWith(allowed) : h === allowed + allowed.startsWith('.') + ? h === allowed.slice(1) || + new RegExp( + `^(?:[a-z0-9-]+\\.)+${allowed.slice(1).replace(/\./g, '\\.')}$` + ).test(h) + : h === allowed ); } +/** Code points, as the database's char_length counts them. */ +const codePoints = (v: string) => Array.from(v).length; + +/** An unpaired UTF-16 surrogate, which Postgres jsonb cannot store. */ +const LONE_SURROGATE_RE = + /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? EXTENSION_OBSERVATION_LIMITS.urlMax || - v.includes('?') || - v.includes('#') || - CONTROL_RE.test(v) + codePoints(v) > EXTENSION_OBSERVATION_LIMITS.urlMax || + URL_FORBIDDEN_RE.test(v) || + LONE_SURROGATE_RE.test(v) ) { return false; } - try { - const url = new URL(v); - return ( - url.protocol === 'https:' && - !url.username && - !url.password && - !url.port && - isExtensionMarketplaceHost(marketplace, url.hostname) - ); - } catch { - return false; - } + const host = /^https:\/\/([^/]*)/.exec(v)?.[1]; + return ( + typeof host === 'string' && + /^[a-z0-9.-]+$/.test(host) && + isExtensionMarketplaceHost(marketplace, host) + ); +} + +/** ISO 8601 with a zone, as the RPC requires it. */ +const OBSERVED_AT_RE = + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d{1,9})?)?(?:Z|[+-]\d{2}(?::?\d{2})?)$/; + +function observedAtMs(v: unknown): number { + if (typeof v !== 'string' || !OBSERVED_AT_RE.test(v)) return Number.NaN; + // Date.parse wants the offset as ±hh:mm. + const normalized = v + .replace(/([+-]\d{2})$/, '$1:00') + .replace(/([+-]\d{2})(\d{2})$/, '$1:$2'); + return Date.parse(normalized); } /** - * Checks an observation against v1 at server time `nowMs`. The first field - * that breaks a rule is named, the way the RPC names it in DETAIL. + * Checks an observation against v1 at server time `nowMs`. It refuses what + * the RPC refuses; the field it names is the first rule broken in this order, + * which can differ from the RPC's when several rules break (an unknown key is + * `observation` here and its own name there). */ export function parseExtensionObservationV1( raw: unknown, @@ -203,8 +228,9 @@ export function parseExtensionObservationV1( if ( variant !== null && (typeof variant !== 'string' || - variant.length < 1 || - variant.length > EXTENSION_OBSERVATION_LIMITS.variantMax || + codePoints(variant) < 1 || + LONE_SURROGATE_RE.test(variant) || + codePoints(variant) > EXTENSION_OBSERVATION_LIMITS.variantMax || CONTROL_RE.test(variant)) ) { return fail('variant_label'); @@ -228,10 +254,7 @@ export function parseExtensionObservationV1( ) { return fail('stock_qty'); } - const observedMs = - typeof raw.observed_at === 'string' && raw.observed_at.length <= 40 - ? Date.parse(raw.observed_at) - : Number.NaN; + const observedMs = observedAtMs(raw.observed_at); if ( !Number.isFinite(observedMs) || observedMs > nowMs + EXTENSION_OBSERVATION_LIMITS.futureSkewMs || From d00ca37571b08404c6d497a2ffa88ade19959019 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:38:30 +0700 Subject: [PATCH 3/4] fix(web): observed_at bounds match postgres, brake retry-after - observed_at refuses seconds of 60 and offsets beyond +-15:59, which Postgres refuses, so the validator and the RPC agree. - A 54000 from the global register brake answers Retry-After 60; the hourly limits keep 3600. Refs #1606 Co-Authored-By: Claude Opus 5.5 --- .../src/app/api/v1/rate/__tests__/write-paths.test.ts | 10 ++++++++++ apps/web/src/server/rate/extension-observations.ts | 5 +++-- packages/shared-data/src/__tests__/shared-data.test.ts | 2 ++ .../shared-data/src/rules/extension-observation.ts | 3 ++- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts index c4e040f81..d1d204135 100644 --- a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts +++ b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts @@ -1072,6 +1072,16 @@ async function testExtensionObservations() { assert.equal(res.headers.get('retry-after'), '3600'); } } + // The global register brake is per minute. + rpcResponder = () => ({ + status: 400, + json: { code: '54000', details: 'submitter' }, + }); + const brake = await submitterRoute.POST( + post('/api/v1/rate/extension/submitter', {}) + ); + assert.equal(brake.status, 429); + assert.equal(brake.headers.get('retry-after'), '60'); // Deleting answers the same for any id and calls the delete RPC. calls = []; diff --git a/apps/web/src/server/rate/extension-observations.ts b/apps/web/src/server/rate/extension-observations.ts index 9a81622e8..8259c90cc 100644 --- a/apps/web/src/server/rate/extension-observations.ts +++ b/apps/web/src/server/rate/extension-observations.ts @@ -211,11 +211,12 @@ export function rpcFailureResponse( }); if (isMissingRpcError(error)) return unavailableResponse(request); if (error.code === '54000') { - // The database limits are hourly. + // The per-submitter and per-network limits are hourly; the global + // register brake (DETAIL "submitter") is per minute. return rateLimitExceededResponse( 'Too many requests; try again later.', observationHeaders(request), - 3600 + error.details === 'submitter' ? 60 : 3600 ); } if (error.code === 'P0002') { diff --git a/packages/shared-data/src/__tests__/shared-data.test.ts b/packages/shared-data/src/__tests__/shared-data.test.ts index 5b6acb9a6..f008e4d4b 100644 --- a/packages/shared-data/src/__tests__/shared-data.test.ts +++ b/packages/shared-data/src/__tests__/shared-data.test.ts @@ -2397,6 +2397,8 @@ function testExtensionObservation(): void { 'listing_url', ], [{ ...valid, variant_label: 'a\ud800b' }, 'variant_label'], + [{ ...valid, observed_at: '2026-11-04T10:00:00+16:00' }, 'observed_at'], + [{ ...valid, observed_at: '2026-11-04T09:59:60Z' }, 'observed_at'], [{ ...valid, observed_at: '2026-11-04' }, 'observed_at'], [{ ...valid, observed_at: '2026-11-04T09:59' }, 'observed_at'], [{ ...valid, observed_at: '2026-11-04 09:59Z' }, 'observed_at'], diff --git a/packages/shared-data/src/rules/extension-observation.ts b/packages/shared-data/src/rules/extension-observation.ts index 9d65f70ea..ea739d1e9 100644 --- a/packages/shared-data/src/rules/extension-observation.ts +++ b/packages/shared-data/src/rules/extension-observation.ts @@ -187,8 +187,9 @@ function listingUrlOk(v: unknown, marketplace: ExtensionMarketplace): boolean { } /** ISO 8601 with a zone, as the RPC requires it. */ +// Seconds stop at 59 and offsets at ±15:59, the range Postgres accepts. const OBSERVED_AT_RE = - /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d{1,9})?)?(?:Z|[+-]\d{2}(?::?\d{2})?)$/; + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::[0-5]\d(?:\.\d{1,9})?)?(?:Z|[+-](?:0\d|1[0-5])(?::?[0-5]\d)?)$/; function observedAtMs(v: unknown): number { if (typeof v !== 'string' || !OBSERVED_AT_RE.test(v)) return Number.NaN; From 36683b36be7bc48db9650b283e232cc4475b1ab5 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:07:53 +0700 Subject: [PATCH 4/4] fix(shared-data,web): host check without a built regex, test key at run time - isExtensionMarketplaceHost checks subdomains by suffix and a fixed label pattern instead of a RegExp built from the host table (CodeQL incomplete escaping). - The ingest key test value is built at run time; the already-pushed fixture line in d19083ac gets a .gitleaksignore fingerprint. Refs #1606 Co-Authored-By: Claude Opus 5.5 --- .gitleaksignore | 3 +++ .../api/v1/rate/__tests__/write-paths.test.ts | 3 ++- .../src/rules/extension-observation.ts | 19 +++++++++++-------- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.gitleaksignore b/.gitleaksignore index a74366ea9..c293aac92 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -11,3 +11,6 @@ 2f624a1c620f68ecb658eed02aba62b05f05b04b:apps/web/src/lib/review-form-v2.ts:generic-api-key:193 4c0506053bbbc6668df016cdebfd3ee37fc9cea8:apps/web/src/lib/review-form-v2.ts:generic-api-key:75 4c0506053bbbc6668df016cdebfd3ee37fc9cea8:apps/web/src/lib/review-form-v2.ts:generic-api-key:76 +# Test fixture of the extension ingest key in write-paths.test.ts (#1606), a fake value read +# as a generic API key in an already-pushed commit. Later commits build it at run time. +d19083ac00645e6296031283025b5dd751eaab78:apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts:generic-api-key:894 diff --git a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts index d1d204135..39f2e294d 100644 --- a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts +++ b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts @@ -891,7 +891,8 @@ async function testExtensionObservations() { process.env.RATE_EXTENSION_OBSERVATIONS = '1'; const savedSalt = process.env.RATE_NETWORK_HASH_SALT; const savedKey = process.env.RATE_EXTENSION_INGEST_KEY; - const INGEST_KEY = 'ingest-key-0123456789abcdef0123456789'; + // A test value built at run time, so it never reads as a secret. + const INGEST_KEY = ['ingest', 'test', 'key'].join('-').padEnd(40, 'x'); try { // On, but without the salt or the ingest key: writes are refused before // the database, since its limits and key check would not hold. diff --git a/packages/shared-data/src/rules/extension-observation.ts b/packages/shared-data/src/rules/extension-observation.ts index ea739d1e9..714a5ae14 100644 --- a/packages/shared-data/src/rules/extension-observation.ts +++ b/packages/shared-data/src/rules/extension-observation.ts @@ -147,14 +147,17 @@ export function isExtensionMarketplaceHost( host: string ): boolean { const h = host.toLowerCase(); - return EXTENSION_MARKETPLACE_HOSTS[marketplace].some((allowed) => - allowed.startsWith('.') - ? h === allowed.slice(1) || - new RegExp( - `^(?:[a-z0-9-]+\\.)+${allowed.slice(1).replace(/\./g, '\\.')}$` - ).test(h) - : h === allowed - ); + return EXTENSION_MARKETPLACE_HOSTS[marketplace].some((allowed) => { + if (!allowed.startsWith('.')) return h === allowed; + // A subdomain: non-empty labels of [a-z0-9-] before the base domain, + // as the RPC's ^([a-z0-9-]+\.)*base$ requires. No pattern is built + // from data. + if (h === allowed.slice(1)) return true; + if (!h.endsWith(allowed)) return false; + return /^(?:[a-z0-9-]+\.)+$/.test( + h.slice(0, h.length - allowed.length + 1) + ); + }); } /** Code points, as the database's char_length counts them. */