From 95c6d5499fd3dae41cc3b806f07896e430165176 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:59:22 +0700 Subject: [PATCH 1/2] feat(shared-data): per-review author key for seeding analysis (#1615) * feat(shared-data): per-review author key for duplicate-text analysis Rate-Services will mask stored author names (RS#1073), so the analyzer can no longer tell authors apart by author_name. Reviews may now carry an optional authorKey (keyed HMAC computed by RS, 16 to 128 chars of [A-Za-z0-9_-]). - ScrapedReviewSchema and ShopeeReview accept authorKey. - The importer stores it in rate.review_author_keys (a table anon cannot read, unlike rate.reviews) and loads it for the analysis; failures log a warning. - Duplicate-text and near-duplicate seeding checks group by author_key when present, else by author_name, else by review id. - Proposal SQL, verify, rollback and the ingest contract in Docs/Proposals. Closes #1614 * fix(shared-data): track proposal SQL, all-or-nothing key load, drop bad keys - Force-add the three proposal SQL files (git-ignored by *.sql). - loadAuthorKeys returns an empty map on any failure so one analysis never mixes key and name identities. - A malformed authorKey is dropped instead of rejecting the listing. - Document the mixed-identity limits. * chore: allow the author-keys proposal SQL in .gitignore Refs #1614 * docs(proposals): author-keys migration after the second review Worker policy limited to crawled reviews, lock_timeout, no unused index, privilege-matrix verify, and the secret-rotation limit in the contract. Refs #1614 * fix(shared-data): leave database.types.ts to the migration mirror gen-db-types --check compares the file with supabase/migrations, and the review_author_keys migration is still a DOS-Me proposal. The generated type arrives with the mirror once DOS-Me applies it. Refs #1614 Co-Authored-By: Claude Sonnet 5.5 --- .gitignore | 4 + ...-06-dos-me-review-author-keys-rollback.sql | 5 ++ ...11-06-dos-me-review-author-keys-verify.sql | 26 ++++++ .../2026-11-06-dos-me-review-author-keys.md | 56 ++++++++++++ .../2026-11-06-dos-me-review-author-keys.sql | 55 ++++++++++++ .../__tests__/shopee-to-supabase.test.ts | 41 ++++++++- .../importer/src/shopee/ai-review-analyzer.ts | 27 +++++- .../importer/src/shopee/shopee-to-supabase.ts | 87 +++++++++++++++++++ .../src/shopee/test-review-analyzer.ts | 38 ++++++++ apps/importer/src/shopee/types.ts | 2 + .../src/__tests__/shared-data.test.ts | 38 ++++++++ .../shared-data/src/schemas/specs.schema.ts | 7 ++ 12 files changed, 382 insertions(+), 4 deletions(-) create mode 100644 Docs/Proposals/2026-11-06-dos-me-review-author-keys-rollback.sql create mode 100644 Docs/Proposals/2026-11-06-dos-me-review-author-keys-verify.sql create mode 100644 Docs/Proposals/2026-11-06-dos-me-review-author-keys.md create mode 100644 Docs/Proposals/2026-11-06-dos-me-review-author-keys.sql diff --git a/.gitignore b/.gitignore index 3e5f451ed..4e581e9fc 100644 --- a/.gitignore +++ b/.gitignore @@ -187,3 +187,7 @@ storageState*.json !*.env.js !*.env.mjs !*.env.d.ts +# Reviewed DOS-Me migration proposal (DOS/Rate#1614): DDL only, no data, no credentials. +!Docs/Proposals/2026-11-06-dos-me-review-author-keys.sql +!Docs/Proposals/2026-11-06-dos-me-review-author-keys-rollback.sql +!Docs/Proposals/2026-11-06-dos-me-review-author-keys-verify.sql diff --git a/Docs/Proposals/2026-11-06-dos-me-review-author-keys-rollback.sql b/Docs/Proposals/2026-11-06-dos-me-review-author-keys-rollback.sql new file mode 100644 index 000000000..b6ed07d5a --- /dev/null +++ b/Docs/Proposals/2026-11-06-dos-me-review-author-keys-rollback.sql @@ -0,0 +1,5 @@ +-- ROLLBACK the PROPOSAL 2026-11-06-dos-me-review-author-keys.sql (DOS/Rate#1614). +-- Safe to run twice. The analyzer then groups authors by author_name again. +BEGIN; +DROP TABLE IF EXISTS rate.review_author_keys; +COMMIT; diff --git a/Docs/Proposals/2026-11-06-dos-me-review-author-keys-verify.sql b/Docs/Proposals/2026-11-06-dos-me-review-author-keys-verify.sql new file mode 100644 index 000000000..2b7e94c7c --- /dev/null +++ b/Docs/Proposals/2026-11-06-dos-me-review-author-keys-verify.sql @@ -0,0 +1,26 @@ +-- VERIFY the PROPOSAL 2026-11-06-dos-me-review-author-keys.sql (DOS/Rate#1614). +-- Read-only. V1 to V4 return 0. + +-- V1 the table exists with RLS on; expect 0 +SELECT count(*) FROM (SELECT 1) x +WHERE NOT EXISTS ( + SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'rate' AND c.relname = 'review_author_keys' AND c.relrowsecurity +); + +-- V2 privilege matrix: anon and authenticated have nothing, rate_worker has no DELETE or +-- TRUNCATE; expect 0 +SELECT count(*) FROM (VALUES + ('anon', 'SELECT'), ('anon', 'INSERT'), ('anon', 'UPDATE'), ('anon', 'DELETE'), + ('authenticated', 'SELECT'), ('authenticated', 'INSERT'), ('authenticated', 'UPDATE'), + ('authenticated', 'DELETE'), ('rate_worker', 'DELETE'), ('rate_worker', 'TRUNCATE') +) AS t(role_name, priv) +WHERE has_table_privilege(t.role_name, 'rate.review_author_keys', t.priv) + OR has_any_column_privilege(t.role_name, 'rate.review_author_keys', t.priv); + +-- V3 no stored key has a bad shape; expect 0 +SELECT count(*) FROM rate.review_author_keys WHERE author_key !~ '^[A-Za-z0-9_-]{16,128}$'; + +-- V4 no key row points at a missing review; expect 0 +SELECT count(*) FROM rate.review_author_keys k +WHERE NOT EXISTS (SELECT 1 FROM rate.reviews r WHERE r.id = k.review_id); diff --git a/Docs/Proposals/2026-11-06-dos-me-review-author-keys.md b/Docs/Proposals/2026-11-06-dos-me-review-author-keys.md new file mode 100644 index 000000000..9268c0076 --- /dev/null +++ b/Docs/Proposals/2026-11-06-dos-me-review-author-keys.md @@ -0,0 +1,56 @@ +# Per-review author key (DOS/Rate#1614) + +Status: **proposal, nothing applied.** The migration goes to DOS-Me; DOS/Rate never applies it. + +Rate-Services (DOS/Rate-Services#1073) masks `rate.reviews.author_name` and redacts contact data +before the engine inserts reviews. The analyzer told authors apart by `author_name` for the +duplicate-text and seeding checks, so masked names would merge many real authors. RS therefore +sends a key per review and the analyzer groups by it. + +## Ingest contract (what RS sends) + +On each review of a `ScrapedProduct` (`reviews[]`): + +| Field | Type | Rule | +| ----------- | ------ | ------------------------------------------------------------------------------- | +| `authorKey` | string | optional; 16 to 128 chars of `[A-Za-z0-9_-]`, else dropped (the review is kept) | + +- Recommended value: hex of `HMAC-SHA256(secret, normalised original author name)` (64 chars). + The secret stays in RS; Rate never sees it. Use one secret for all products and platforms so + the same author keeps one key. Not reversible. +- The key is compared exactly (case-sensitive). Keep the normalisation in RS stable (for example + NFC, trimmed, lower-cased) so one author gets one key. +- Absent key: the analyzer groups by `author_name` as before. Old rows keep working. +- `author_name` is stored and shown as sent (masked or not). No UI change. + +## Storage + +`rate.review_author_keys (review_id, author_key, created_at)`, not a column on `rate.reviews`: +anon has a table-level SELECT on `rate.reviews`, and a stable key would let anyone link one +author's reviews across products. The new table is readable only by `service_role` and +`rate_worker`. The importer upserts a row per keyed review after inserting the reviews (also for +reviews already stored, so a later crawl backfills) and reads them for the analysis. A write or +read failure logs a warning and the analysis falls back to `author_name`. + +## Analyzer + +`reviewAuthorIdentity` (apps/importer `ai-review-analyzer.ts`): `key:` when present, +else `name:`, else `id:`. Duplicate-text counts and the +near-duplicate shingle pass use it. Scoring weights are unchanged. + +Limits: reviews stored before masking carry a raw name and get a key only when crawled again; +until then they are one identity by name and a keyed copy of the same author counts as another. +Unkeyed masked names ("N***n") can merge different authors. RS should send a key on every review. +Do not rotate the HMAC secret or change the name normalisation: a crawl returns only a window of +recent reviews, so old and new keys of one author would meet in one analysis and count as two +authors. A change needs a full re-send of every review first. +The worker role can write keys only for crawled reviews (not `origin = 'community'`). +Also: `rate.review_input_hash` includes `author_name`, so masking changes that hash (RS/DB side; the crawl path does not use it). + +## Files + +| File | What | +| --------------------------------------------------- | ------------------------- | +| `2026-11-06-dos-me-review-author-keys.sql` | the migration | +| `2026-11-06-dos-me-review-author-keys-verify.sql` | read-only checks V1 to V4 | +| `2026-11-06-dos-me-review-author-keys-rollback.sql` | drops the table | diff --git a/Docs/Proposals/2026-11-06-dos-me-review-author-keys.sql b/Docs/Proposals/2026-11-06-dos-me-review-author-keys.sql new file mode 100644 index 000000000..165b9044b --- /dev/null +++ b/Docs/Proposals/2026-11-06-dos-me-review-author-keys.sql @@ -0,0 +1,55 @@ +-- ============================================================================= +-- PROPOSAL (DOS/Rate#1614): a per-review author key, so the review analyzer can tell authors +-- apart after Rate-Services masks rate.reviews.author_name (DOS/Rate-Services#1073). A DOS-Me +-- migration to review and apply; nothing here is applied by DOS/Rate. +-- Verify: 2026-11-06-dos-me-review-author-keys-verify.sql (V1 to V4 return 0). +-- Rollback: 2026-11-06-dos-me-review-author-keys-rollback.sql. +-- +-- What it adds: +-- rate.review_author_keys one row per review that has a key: the keyed HMAC of the original +-- author name, computed by Rate-Services (the key never reaches +-- Rate). It is a separate table on purpose: anon holds a table-level +-- SELECT on rate.reviews, so a new column there would be public, and +-- a stable key would let anyone link one author's reviews across +-- products. This table has no grant for anon or authenticated. +-- +-- The importer (service_role, rate_worker) writes it and reads it for the analysis. Reviews +-- without a row are analysed by author_name as before. Nothing the user sees changes. +-- +-- Idempotent. One transaction. +-- ============================================================================= + +BEGIN; +SET LOCAL lock_timeout = '5s'; + +CREATE TABLE IF NOT EXISTS rate.review_author_keys ( + review_id uuid PRIMARY KEY REFERENCES rate.reviews (id) ON DELETE CASCADE, + -- hex or base64url of an HMAC: 16 to 128 characters + author_key text NOT NULL CHECK (author_key ~ '^[A-Za-z0-9_-]{16,128}$'), + created_at timestamptz NOT NULL DEFAULT now() +); + +ALTER TABLE rate.review_author_keys ENABLE ROW LEVEL SECURITY; +REVOKE ALL ON rate.review_author_keys FROM PUBLIC, anon, authenticated, rate_worker; +GRANT ALL ON rate.review_author_keys TO service_role; +GRANT SELECT, INSERT, UPDATE ON rate.review_author_keys TO rate_worker; + +DROP POLICY IF EXISTS review_author_keys_service ON rate.review_author_keys; +CREATE POLICY review_author_keys_service ON rate.review_author_keys + FOR ALL TO service_role USING (true) WITH CHECK (true); +-- Like reviews and review_flags: the worker reads all, writes only keys of crawled reviews. +DROP POLICY IF EXISTS review_author_keys_worker ON rate.review_author_keys; +DROP POLICY IF EXISTS review_author_keys_worker_read ON rate.review_author_keys; +DROP POLICY IF EXISTS review_author_keys_worker_insert ON rate.review_author_keys; +DROP POLICY IF EXISTS review_author_keys_worker_update ON rate.review_author_keys; +CREATE POLICY review_author_keys_worker_read ON rate.review_author_keys + FOR SELECT TO rate_worker USING (true); +CREATE POLICY review_author_keys_worker_insert ON rate.review_author_keys + FOR INSERT TO rate_worker + WITH CHECK (EXISTS (SELECT 1 FROM rate.reviews r WHERE r.id = review_author_keys.review_id AND r.origin <> 'community')); +CREATE POLICY review_author_keys_worker_update ON rate.review_author_keys + FOR UPDATE TO rate_worker + USING (EXISTS (SELECT 1 FROM rate.reviews r WHERE r.id = review_author_keys.review_id AND r.origin <> 'community')) + WITH CHECK (EXISTS (SELECT 1 FROM rate.reviews r WHERE r.id = review_author_keys.review_id AND r.origin <> 'community')); + +COMMIT; diff --git a/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts b/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts index 68826178d..c8952eeb1 100644 --- a/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts +++ b/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts @@ -53,6 +53,7 @@ function createBaseProduct(overrides: Partial = {}): ShopeeProduc } interface StubPipelineOptions { + storedAuthorKeys?: Array<{ review_id: string; author_key: string }>; categoriesErr?: { message: string } | null; hintRow?: { id: string; storage_gb?: number | null } | null; hintErr?: { message: string } | null; @@ -83,6 +84,7 @@ function createPipelineStubSupabase(opts: StubPipelineOptions = {}) { reviewInserts: Array>>; verifiedUpdates: Array<{ verified: boolean; ids: string[] }>; marketStatsRefreshed: string[]; + authorKeyUpserts: Array>>; } = { merchantUpserts: 0, merchantPayloads: [], @@ -93,6 +95,7 @@ function createPipelineStubSupabase(opts: StubPipelineOptions = {}) { reviewInserts: [], verifiedUpdates: [], marketStatsRefreshed: [], + authorKeyUpserts: [], }; let reviewSelectPass = 0; @@ -253,6 +256,20 @@ function createPipelineStubSupabase(opts: StubPipelineOptions = {}) { if (table === 'reviews') { return { select(cols: string) { + if (cols === 'id, platform_review_id') { + return { + eq(_col: string, _val: string) { + return { + in(_inCol: string, requested: string[]) { + return Promise.resolve({ + data: requested.map(id => ({ id: `db-${id}`, platform_review_id: id })), + error: null, + }); + }, + }; + }, + }; + } if (cols === 'platform_review_id') { return { eq(_col: string, _val: string) { @@ -357,6 +374,22 @@ function createPipelineStubSupabase(opts: StubPipelineOptions = {}) { }; } + if (table === 'review_author_keys') { + return { + select(_columns: string) { + return { + in(_column: string, _ids: unknown[]) { + return Promise.resolve({ data: opts.storedAuthorKeys ?? [], error: null }); + }, + }; + }, + upsert(payload: Array>, _options?: unknown) { + events.authorKeyUpserts.push(payload); + return Promise.resolve({ data: null, error: null }); + }, + }; + } + if (table === 'review_flags' || table === 'review_analysis') { return { // persistAnalysisToSupabase reads flags written by other detectors first. @@ -818,7 +851,8 @@ async function runTests() { }, { reviewId: 'rev-explicit-1', - author: 'Minh Anh', + author: 'M***h', + authorKey: 'c'.repeat(64), ratingStar: 5, comment: 'Review mới còn lại sau khi lọc race', images: [], @@ -846,6 +880,11 @@ async function runTests() { 'Pass 2 should insert only the remaining non-raced review' ); assert.equal(revRaceEvents.reviewInserts[1]?.[0]?.platform_review_id, 'rev-explicit-1'); + assert.deepEqual( + revRaceEvents.authorKeyUpserts, + [[{ review_id: 'db-rev-explicit-1', author_key: 'c'.repeat(64) }]], + 'author_key from RS is stored in review_author_keys for the review (DOS/Rate#1614)' + ); assert.equal( revRaceEvents.verifiedUpdates.length, 2, diff --git a/apps/importer/src/shopee/ai-review-analyzer.ts b/apps/importer/src/shopee/ai-review-analyzer.ts index 1f750546f..ddd0de5a6 100644 --- a/apps/importer/src/shopee/ai-review-analyzer.ts +++ b/apps/importer/src/shopee/ai-review-analyzer.ts @@ -134,6 +134,11 @@ export interface BayesianRatingResult { export interface ReviewInputItem { id: string; author_name?: string | null; + /** + * Keyed HMAC of the original author name, computed by Rate-Services before it masks + * `author_name`. Preferred over the name to tell authors apart; absent on old rows. + */ + author_key?: string | null; rating_star: number; comment?: string | null; variation?: string | null; @@ -148,6 +153,24 @@ export interface ReviewInputItem { reviewer?: ScrapedReviewer | null; } +/** + * Identity used to tell authors apart for duplicate-text and seeding analysis: the + * `author_key` when present (exact, case-sensitive), else the lower-cased `author_name`, + * else the review id. Prefixes keep a key from ever equal to a name. + */ +export function reviewAuthorIdentity( + r: Pick, + reviewId: string +): string { + if (typeof r.author_key === 'string' && r.author_key.trim()) { + return `key:${r.author_key.trim()}`; + } + if (typeof r.author_name === 'string' && r.author_name.trim()) { + return `name:${r.author_name.toLowerCase()}`; + } + return `id:${reviewId}`; +} + export interface AnalyzedReviewResult { review_id: string; verdict: ReviewVerdict; @@ -1182,9 +1205,7 @@ export class AiReviewAnalyzer { const norm = normalizeTextForDedupe(r.comment || ''); // Rating labels repeat across buyers because the platform writes them, not because of seeding. if (norm.length >= 15 && !isRatingOnlyComment(r.comment)) { - const rawAuthor = - typeof r.author_name === 'string' && r.author_name.trim() ? r.author_name : reviewId; - const authorKey = rawAuthor.toLowerCase(); + const authorKey = reviewAuthorIdentity(r, reviewId); const authors = commentCounts.get(norm) || new Set(); authors.add(authorKey); commentCounts.set(norm, authors); diff --git a/apps/importer/src/shopee/shopee-to-supabase.ts b/apps/importer/src/shopee/shopee-to-supabase.ts index 7f4533247..80ae3549f 100644 --- a/apps/importer/src/shopee/shopee-to-supabase.ts +++ b/apps/importer/src/shopee/shopee-to-supabase.ts @@ -402,6 +402,85 @@ export class ShopeeIngestionPipeline { * request, a short URL) instead of reading every stored review, which * PostgREST caps at max-rows. */ + /** + * Keep the per-review author key RS computed (DOS/Rate#1614) in rate.review_author_keys, + * a table anon cannot read. Never fails the ingest: without a key the analyzer falls back + * to author_name. + */ + private async storeAuthorKeys( + productId: string, + reviews: readonly ShopeeReview[] + ): Promise { + const keyByPlatformId = new Map(); + for (const r of reviews) { + const platformId = platformReviewIdFor(r); + if (platformId && typeof r.authorKey === 'string' && r.authorKey) { + keyByPlatformId.set(platformId, r.authorKey); + } + } + if (keyByPlatformId.size === 0) return; + try { + const platformIds = [...keyByPlatformId.keys()]; + for (let i = 0; i < platformIds.length; i += 50) { + const chunk = platformIds.slice(i, i + 50); + const { data: rows, error: lookupErr } = await this.supabase + .from('reviews') + .select('id, platform_review_id') + .eq('product_id', productId) + .in('platform_review_id', chunk); + if (lookupErr) throw new Error(lookupErr.message); + const payload = (rows || []) + .map(row => ({ + review_id: row.id as string, + author_key: keyByPlatformId.get(row.platform_review_id as string), + })) + .filter(p => Boolean(p.review_id && p.author_key)); + if (payload.length === 0) continue; + const { error: upsertErr } = await this.supabase + .from('review_author_keys') + .upsert(payload, { onConflict: 'review_id' }); + if (upsertErr) throw new Error(upsertErr.message); + } + } catch (err) { + logger.warn( + 'ingest.author_keys_store_warn', + `Could not store review author keys: ${err instanceof Error ? err.message : String(err)}`, + { productId } + ); + } + } + + /** Author keys of stored reviews by review id; empty when none or the read fails. */ + private async loadAuthorKeys( + productId: string, + reviewIds: readonly string[] + ): Promise> { + let out = new Map(); + const ids = reviewIds.filter(Boolean); + try { + for (let i = 0; i < ids.length; i += 100) { + const { data, error } = await this.supabase + .from('review_author_keys') + .select('review_id, author_key') + .in('review_id', ids.slice(i, i + 100)); + if (error) throw new Error(error.message); + for (const row of data || []) { + if (row.review_id && row.author_key) + out.set(String(row.review_id), String(row.author_key)); + } + } + } catch (err) { + // All or nothing: a partial map would mix key and name identities in one analysis. + out = new Map(); + logger.warn( + 'ingest.author_keys_load_warn', + `Review analysis falls back to author_name: ${err instanceof Error ? err.message : String(err)}`, + { productId } + ); + } + return out; + } + private async findExistingReviewIds( productId: string, ids: readonly string[], @@ -1112,6 +1191,7 @@ export class ShopeeIngestionPipeline { ); } } + await this.storeAuthorKeys(productId, product.reviews); // Rows stored before the platform's purchase flag was captured still carry the // column default; refresh it for reviews already in the DB. for (const verified of [true, false]) { @@ -1169,10 +1249,17 @@ export class ShopeeIngestionPipeline { .filter(r => r.reviewer) .map(r => [platformReviewIdFor(r), r.reviewer] as const) ); + const authorKeyByReviewId = storedErr + ? new Map() + : await this.loadAuthorKeys( + productId, + (storedReviews || []).map(r => String(r?.id ?? '')) + ); const analysisInput: ReviewInputItem[] = (storedReviews || []) .filter(row => Boolean(row && typeof row === 'object' && !Array.isArray(row))) .map((row: ReviewInputItem & { platform_review_id?: string | null }) => ({ ...row, + author_key: authorKeyByReviewId.get(String(row.id)) ?? null, reviewer: row.platform_review_id ? reviewerByPlatformId.get(row.platform_review_id) : undefined, diff --git a/apps/importer/src/shopee/test-review-analyzer.ts b/apps/importer/src/shopee/test-review-analyzer.ts index 3dd224787..0e2df6dad 100644 --- a/apps/importer/src/shopee/test-review-analyzer.ts +++ b/apps/importer/src/shopee/test-review-analyzer.ts @@ -1220,6 +1220,44 @@ async function run() { assert.equal(copiedBatch[1].flag_category, 'seeding_fake'); assert.equal(copiedBatch[1].assessment?.authenticity, 'copied'); + // 9.3b Per-review author key (DOS/Rate#1614): RS masks author_name, the key tells authors apart. + const keyedText = + 'Máy nguyên seal chính hãng dùng cực kỳ mượt mà, pin trâu dùng cả ngày không nóng'; + const keyA = 'a'.repeat(64); + const keyB = 'b'.repeat(64); + const sameMaskDiffKey = analyzer.analyzeWithHeuristics([ + { id: 'mk-1', author_name: 'N***n', author_key: keyA, rating_star: 5, comment: keyedText }, + { id: 'mk-2', author_name: 'N***n', author_key: keyB, rating_star: 5, comment: keyedText }, + ]); + assert.equal( + sameMaskDiffKey[0].flag_category, + 'seeding_fake', + 'same masked name but different author_key must count as two authors' + ); + const sameKey = analyzer.analyzeWithHeuristics([ + { id: 'mk-3', author_name: 'N***n', author_key: keyA, rating_star: 5, comment: keyedText }, + { id: 'mk-4', author_name: 'Q***x', author_key: keyA, rating_star: 5, comment: keyedText }, + ]); + assert.notEqual( + sameKey[0].flag_category, + 'seeding_fake', + 'same author_key is one author even when the masked names differ' + ); + const noKey = analyzer.analyzeWithHeuristics([ + { id: 'mk-5', author_name: 'N***n', rating_star: 5, comment: keyedText }, + { id: 'mk-6', author_name: 'N***n', rating_star: 5, comment: keyedText }, + ]); + assert.notEqual( + noKey[0].flag_category, + 'seeding_fake', + 'without author_key the author_name fallback applies (same name, one author)' + ); + const nameDiff = analyzer.analyzeWithHeuristics([ + { id: 'mk-7', author_name: 'seeder_a', rating_star: 5, comment: keyedText }, + { id: 'mk-8', author_name: 'seeder_b', rating_star: 5, comment: keyedText }, + ]); + assert.equal(nameDiff[0].flag_category, 'seeding_fake', 'name fallback still separates authors'); + // 9.4 Affiliate Shortlink Detection (Section 3 Axis B / P2) const affAssess = assessReview6Axes({ id: 'aff-shortlink', diff --git a/apps/importer/src/shopee/types.ts b/apps/importer/src/shopee/types.ts index 6c52d4457..7fb2a5adf 100644 --- a/apps/importer/src/shopee/types.ts +++ b/apps/importer/src/shopee/types.ts @@ -71,6 +71,8 @@ export interface ScrapedReviewer { export interface ShopeeReview { reviewId: string; author: string; + /** Keyed HMAC of the original author name (RS), 16 to 128 chars of [A-Za-z0-9_-]. */ + authorKey?: string; authorAvatar?: string; ratingStar: number; comment: string; diff --git a/packages/shared-data/src/__tests__/shared-data.test.ts b/packages/shared-data/src/__tests__/shared-data.test.ts index f008e4d4b..272d0a7d4 100644 --- a/packages/shared-data/src/__tests__/shared-data.test.ts +++ b/packages/shared-data/src/__tests__/shared-data.test.ts @@ -371,6 +371,44 @@ function testMasterProductAndScrapedProductSchemas(): void { false, 'ScrapedProductSchema must reject non-http reviews[].authorAvatar' ); + for (const [key, ok] of [ + ['a'.repeat(64), true], + ['Ab_-0123456789xyz', true], + ['short', false], + ['a'.repeat(129), false], + ['has space in the key!', false], + ] as const) { + const parsed = ScrapedProductSchema.safeParse({ + platform: 'shopee', + itemId: '123456', + shopId: '654321', + url: 'https://shopee.vn/product/654321/123456', + title: 'iPhone 16 Pro Max 256GB', + priceMin: 31990000, + priceMax: 33990000, + reviews: [ + { + reviewId: 'r1', + author: 'B***r', + authorKey: key, + ratingStar: 5, + comment: 'Good', + }, + ], + shop: { shopId: '654321', name: 'Apple Flagship Store', isMall: true }, + }); + assert.equal( + parsed.success, + true, + 'a malformed authorKey must not reject the listing' + ); + + assert.equal( + parsed.success ? parsed.data.reviews[0]?.authorKey : 'x', + ok ? key : undefined, + `authorKey ${key.slice(0, 12)} is ${ok ? 'kept' : 'dropped'}` + ); + } assert.equal( ScrapedProductSchema.safeParse({ platform: 'shopee', diff --git a/packages/shared-data/src/schemas/specs.schema.ts b/packages/shared-data/src/schemas/specs.schema.ts index df92aedae..8d4eedc70 100644 --- a/packages/shared-data/src/schemas/specs.schema.ts +++ b/packages/shared-data/src/schemas/specs.schema.ts @@ -205,6 +205,13 @@ export const ScrapedReviewerSchema = z.object({ export const ScrapedReviewSchema = z.object({ reviewId: z.string().max(128), author: z.string().max(200), + /** Keyed HMAC of the original author name, computed by Rate-Services; not reversible. */ + authorKey: z + .string() + .regex(/^[A-Za-z0-9_-]{16,128}$/) + .optional() + // A malformed key is dropped, never a reason to reject the listing: analysis uses author_name. + .catch(undefined), authorAvatar: HttpUrlSchema.optional(), ratingStar: z.number().min(1).max(5), comment: z.string().max(10_000), From ef404671199fd4af0b319ea19679013baad9a930 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:44:48 +0700 Subject: [PATCH 2/2] chore(db): mirror the review author keys migration from dos-me (#1616) * chore(db): mirror the review author keys migration from dos-me Mirrors 20261109130000_rate_review_author_keys.sql byte for byte (DOS.Me #1584, promote DOS.Me #1585) and regenerates database.types.ts. Refs #1614 Co-Authored-By: Claude Sonnet 5.5 --- packages/shared-data/src/database.types.ts | 25 +++++++++ ...20261109130000_rate_review_author_keys.sql | 55 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 supabase/migrations/20261109130000_rate_review_author_keys.sql diff --git a/packages/shared-data/src/database.types.ts b/packages/shared-data/src/database.types.ts index 1ae3fb041..6f3b41c0e 100644 --- a/packages/shared-data/src/database.types.ts +++ b/packages/shared-data/src/database.types.ts @@ -1050,6 +1050,31 @@ export type Database = { }, ]; }; + review_author_keys: { + Row: { + author_key: string; + created_at: string; + review_id: string; + }; + Insert: { + author_key: string; + created_at?: string; + review_id: string; + }; + Update: { + author_key?: string; + created_at?: string; + review_id?: string; + }; + Relationships: [ + { + foreignKeyName: 'review_author_keys_review_id_fkey'; + columns: ['review_id']; + referencedRelation: 'reviews'; + referencedColumns: ['id']; + }, + ]; + }; review_authors: { Row: { author_user_id: string | null; diff --git a/supabase/migrations/20261109130000_rate_review_author_keys.sql b/supabase/migrations/20261109130000_rate_review_author_keys.sql new file mode 100644 index 000000000..165b9044b --- /dev/null +++ b/supabase/migrations/20261109130000_rate_review_author_keys.sql @@ -0,0 +1,55 @@ +-- ============================================================================= +-- PROPOSAL (DOS/Rate#1614): a per-review author key, so the review analyzer can tell authors +-- apart after Rate-Services masks rate.reviews.author_name (DOS/Rate-Services#1073). A DOS-Me +-- migration to review and apply; nothing here is applied by DOS/Rate. +-- Verify: 2026-11-06-dos-me-review-author-keys-verify.sql (V1 to V4 return 0). +-- Rollback: 2026-11-06-dos-me-review-author-keys-rollback.sql. +-- +-- What it adds: +-- rate.review_author_keys one row per review that has a key: the keyed HMAC of the original +-- author name, computed by Rate-Services (the key never reaches +-- Rate). It is a separate table on purpose: anon holds a table-level +-- SELECT on rate.reviews, so a new column there would be public, and +-- a stable key would let anyone link one author's reviews across +-- products. This table has no grant for anon or authenticated. +-- +-- The importer (service_role, rate_worker) writes it and reads it for the analysis. Reviews +-- without a row are analysed by author_name as before. Nothing the user sees changes. +-- +-- Idempotent. One transaction. +-- ============================================================================= + +BEGIN; +SET LOCAL lock_timeout = '5s'; + +CREATE TABLE IF NOT EXISTS rate.review_author_keys ( + review_id uuid PRIMARY KEY REFERENCES rate.reviews (id) ON DELETE CASCADE, + -- hex or base64url of an HMAC: 16 to 128 characters + author_key text NOT NULL CHECK (author_key ~ '^[A-Za-z0-9_-]{16,128}$'), + created_at timestamptz NOT NULL DEFAULT now() +); + +ALTER TABLE rate.review_author_keys ENABLE ROW LEVEL SECURITY; +REVOKE ALL ON rate.review_author_keys FROM PUBLIC, anon, authenticated, rate_worker; +GRANT ALL ON rate.review_author_keys TO service_role; +GRANT SELECT, INSERT, UPDATE ON rate.review_author_keys TO rate_worker; + +DROP POLICY IF EXISTS review_author_keys_service ON rate.review_author_keys; +CREATE POLICY review_author_keys_service ON rate.review_author_keys + FOR ALL TO service_role USING (true) WITH CHECK (true); +-- Like reviews and review_flags: the worker reads all, writes only keys of crawled reviews. +DROP POLICY IF EXISTS review_author_keys_worker ON rate.review_author_keys; +DROP POLICY IF EXISTS review_author_keys_worker_read ON rate.review_author_keys; +DROP POLICY IF EXISTS review_author_keys_worker_insert ON rate.review_author_keys; +DROP POLICY IF EXISTS review_author_keys_worker_update ON rate.review_author_keys; +CREATE POLICY review_author_keys_worker_read ON rate.review_author_keys + FOR SELECT TO rate_worker USING (true); +CREATE POLICY review_author_keys_worker_insert ON rate.review_author_keys + FOR INSERT TO rate_worker + WITH CHECK (EXISTS (SELECT 1 FROM rate.reviews r WHERE r.id = review_author_keys.review_id AND r.origin <> 'community')); +CREATE POLICY review_author_keys_worker_update ON rate.review_author_keys + FOR UPDATE TO rate_worker + USING (EXISTS (SELECT 1 FROM rate.reviews r WHERE r.id = review_author_keys.review_id AND r.origin <> 'community')) + WITH CHECK (EXISTS (SELECT 1 FROM rate.reviews r WHERE r.id = review_author_keys.review_id AND r.origin <> 'community')); + +COMMIT;