diff --git a/apps/importer/package.json b/apps/importer/package.json index a37c606d..4885316b 100644 --- a/apps/importer/package.json +++ b/apps/importer/package.json @@ -37,10 +37,11 @@ "lint": "eslint . --max-warnings 0", "format": "prettier --write \"src/**/*.ts\"", "backfill:specs": "tsx src/shopee/jobs/backfill-specs.ts", - "test": "tsx src/shopee/__tests__/open-core-intelligence.test.ts && tsx src/shopee/__tests__/sourced-seeds.test.ts && tsx src/shopee/__tests__/entity-matcher.test.ts && tsx src/shopee/__tests__/item-matcher.test.ts && tsx src/shopee/__tests__/shopee-to-supabase.test.ts && tsx src/shopee/__tests__/paging.test.ts && tsx src/shopee/__tests__/backfill-review-risk.test.ts && tsx src/shopee/__tests__/item-inventory-report.test.ts && tsx src/shopee/__tests__/arena-audit.test.ts && tsx src/shopee/__tests__/item-aware-jobs.test.ts && tsx src/shopee/__tests__/product-points.test.ts && tsx src/shopee/test-ai-enricher.ts && tsx src/shopee/test-review-analyzer.ts", + "test": "tsx src/shopee/__tests__/open-core-intelligence.test.ts && tsx src/shopee/__tests__/sourced-seeds.test.ts && tsx src/shopee/__tests__/entity-matcher.test.ts && tsx src/shopee/__tests__/item-matcher.test.ts && tsx src/shopee/__tests__/shopee-to-supabase.test.ts && tsx src/shopee/__tests__/paging.test.ts && tsx src/shopee/__tests__/backfill-review-risk.test.ts && tsx src/shopee/__tests__/item-inventory-report.test.ts && tsx src/shopee/__tests__/arena-audit.test.ts && tsx src/shopee/__tests__/item-aware-jobs.test.ts && tsx src/shopee/__tests__/product-points.test.ts && tsx src/shopee/__tests__/review-values.test.ts && tsx src/shopee/test-ai-enricher.ts && tsx src/shopee/test-review-analyzer.ts", "test:dist": "node src/shopee/__tests__/dist-exports.smoke.mjs", "backfill:review-risk": "tsx src/shopee/jobs/backfill-review-risk.ts", - "product-points": "tsx src/shopee/jobs/product-points.ts" + "product-points": "tsx src/shopee/jobs/product-points.ts", + "review-values": "tsx src/shopee/jobs/review-values.ts" }, "keywords": [ "product-normalizer", diff --git a/apps/importer/src/shopee/__tests__/review-values.test.ts b/apps/importer/src/shopee/__tests__/review-values.test.ts new file mode 100644 index 00000000..4ea89c6c --- /dev/null +++ b/apps/importer/src/shopee/__tests__/review-values.test.ts @@ -0,0 +1,554 @@ +import assert from 'node:assert/strict'; +import { getRatingCriteria, reviewValue } from '@repo/shared-data/rules'; +import { + BATCH_ROWS, + batchValues, + buildReviewValues, + changedValues, + expectedMasterRatings, + masterRatingMatches, + reviewValueInput, + runReviewValuesJob, + storeBatch, + type ReviewValueWrite, + type ValueListingRow, + type ValuePointRow, + type ValueReviewRow, +} from '../jobs/review-values.js'; +import type { RateSupabaseClient } from '../types.js'; + +const PHONES = 'c0000000-0000-4000-8000-000000000001'; +const M1 = 'a0000000-0000-4000-8000-000000000001'; +const M2 = 'a0000000-0000-4000-8000-000000000002'; +const id = (n: number) => `d0000000-0000-4000-8000-${String(n).padStart(12, '0')}`; + +const review = ( + n: number, + product_id: string, + extra: Partial = {} +): ValueReviewRow => ({ + id: id(n), + product_id, + origin: 'crawl', + reviewer_type: 'user', + helpfulness_weight: 1, + rating_star: 4, + rating_exact: null, + comment: null, + verdict: null, + criteria_ratings: null, + images: null, + is_verified_buyer: null, + upvotes: 0, + downvotes: 0, + value_tier: null, + review_flags: { include_in_rating: true, verdict: 'authentic', flag_category: null }, + ...extra, +}); +const line = ( + n: number, + kind: string, + body: string, + criterion_key: string | null = null, + method = 'writer' +): ValuePointRow => ({ + id: `${id(n)}-${(lineNo += 1)}`, + review_id: id(n), + kind, + body, + criterion_key, + method, +}); +let lineNo = 0; + +const categories = [{ id: PHONES, slug: 'smartphones' }]; +const masters = [ + { id: M1, category_id: PHONES }, + { id: M2, category_id: PHONES }, +]; +const listings: ValueListingRow[] = [ + { id: 'l1', cluster_id: M1, origin: 'crawl' }, + { id: 'l2', cluster_id: M2, origin: 'crawl' }, + { id: 'lseed', cluster_id: M1, origin: 'seed' }, +]; +const allRated = Object.fromEntries(getRatingCriteria('smartphones').map(c => [c.key, 4])); +const reviews = [ + // A marketplace review: a detailed comment, a photo, the purchase mark, votes. + review(1, 'l1', { + rating_star: 5, + comment: + 'Pin dùng hết một ngày rưỡi, màn hình sáng rõ ngoài nắng, camera chụp đêm ổn, máy không nóng khi chơi game lâu.', + images: ['https://cf.shopee.vn/file/a.jpg'], + is_verified_buyer: true, + upvotes: 9, + downvotes: 1, + }), + // A Rate review (form v2): no comment, its text is the verdict and the lines. + review(2, 'l1', { + origin: 'community', + rating_exact: 4, + criteria_ratings: allRated, + verdict: 'Máy mượt, pin dùng hết ngày, đáng tiền trong tầm giá.', + }), + // Excluded by the analyzer; not analyzed yet; filler. + review(3, 'l1', { + comment: 'Pin trâu, màn đẹp, chụp ảnh nét, dùng mượt mà không lag.', + review_flags: { include_in_rating: false, verdict: 'suspicious', flag_category: null }, + }), + review(4, 'l1', { comment: 'Pin trâu, màn đẹp.', review_flags: null }), + review(5, 'l1', { + comment: 'Shop giao nhanh, đánh giá để nhận xu nhé.', + review_flags: { + include_in_rating: true, + verdict: 'authentic', + flag_category: 'low_value_coin_spam', + }, + }), + // Seed data is never scored. + review(6, 'l1', { origin: 'seed', comment: 'Pin trâu.' }), + // An expert review Rate published, not analyzed yet: it is not gated. + review(7, 'l2', { + reviewer_type: 'expert', + helpfulness_weight: 0, + comment: 'Đo pin 5000 mAh được 9 giờ màn hình bật.', + review_flags: { include_in_rating: false, verdict: 'unanalyzed', flag_category: null }, + }), + review(8, 'l2', { rating_star: 3, comment: 'Ổn.' }), +]; +const points = [ + line(2, 'pro', 'Pin dùng cả ngày'), + line(2, 'con', 'Sạc hơi chậm'), + line(2, 'pro', 'Chụp đêm rõ nét', 'camera'), + line(2, 'con', 'Màn hơi tối ngoài nắng', 'display'), + // An AI line from a marketplace comment earns nothing. + line(1, 'pro', 'Pin trâu', null, 'ai'), +]; + +async function run() { + console.log('🧪 Review values job suite (#1547)'); + + // Inputs: gates, depth on the whole text, photo, purchase mark and votes. + const input = (n: number) => + reviewValueInput( + reviews.find(r => r.id === id(n))!, + points.filter(p => p.review_id === id(n)), + 'smartphones' + ); + assert.deepEqual( + [1, 2, 3, 4, 5, 7, 8].map(n => input(n).gated), + [false, false, true, true, true, false, false] + ); + assert.equal(input(1).depth, 'detailed', 'a comment on the product'); + assert.equal(input(7).depth, 'expert_depth', 'a measurement'); + assert.equal(input(2).depth, 'detailed', 'the verdict and the lines are its text'); + assert.equal(input(5).depth, 'filler'); + assert.equal(input(1).hasRealPhoto, true); + assert.equal(input(1).verifiedPurchase, true); + assert.equal( + reviewValueInput({ ...reviews[1]!, is_verified_buyer: true }, [], 'smartphones') + .verifiedPurchase, + false, + "a community review's default purchase mark does not count" + ); + assert.deepEqual([input(1).helpfulVotes, input(1).unhelpfulVotes], [9, 1]); + assert.equal(input(1).authorTrust, null); + assert.equal(input(1).points[0]!.method, 'ai'); + assert.equal(input(2).points.length, 4); + + const values = buildReviewValues({ reviews, points, listings, masters, categories }); + assert.deepEqual( + [...values.keys()], + [1, 2, 3, 4, 5, 7, 8].map(id), + 'seed reviews are not scored' + ); + assert.deepEqual(values.get(id(2)), reviewValue(input(2))); + assert.equal(values.get(id(2))!.completeness, 30, 'everything but a photo'); + assert.equal(values.get(id(1))!.weightCompleteness, 0, 'a photo without pros and cons'); + assert.equal(values.get(id(3))!.score, 0); + assert.equal(values.get(id(3))!.tier, null); + + // Only what changed is stored: the number, the breakdown (key order aside) or the tier. + const v1 = values.get(id(1))!; + const reordered = Object.fromEntries(Object.entries(v1).reverse()); + const stored = [ + { review_id: id(1), value_score: v1.score, value_breakdown: reordered }, + { review_id: id(8), value_score: values.get(id(8))!.score, value_breakdown: values.get(id(8)) }, + { review_id: id(2), value_score: 1, value_breakdown: values.get(id(2)) }, + ]; + const tiers = new Map([ + [id(1), v1.tier], + [id(8), 'excellent'], + ]); + assert.deepEqual( + changedValues(values, stored, tiers).map(r => r.id), + [2, 3, 4, 5, 7, 8].map(id), + 'review 1 is unchanged; 8 has a drifted tier; 2 a different number' + ); + + // Batches: at most BATCH_ROWS rows, and a byte limit. + const many: ReviewValueWrite[] = Array.from({ length: BATCH_ROWS * 2 + 1 }, (_, i) => ({ + id: id(i), + score: v1.score, + breakdown: v1, + })); + assert.deepEqual( + batchValues(many).map(b => b.length), + [BATCH_ROWS, BATCH_ROWS, 1] + ); + const big = { + ...v1, + gaps: Array.from({ length: 20_000 }, () => 'photo'), + } as unknown as typeof v1; + const heavy = Array.from({ length: 40 }, (_, i) => ({ id: id(i), score: 1, breakdown: big })); + const heavyBatches = batchValues(heavy); + assert.ok(heavyBatches.length > 1, 'the byte limit splits a batch'); + assert.ok( + heavyBatches.every(b => Buffer.byteLength(JSON.stringify(b)) <= 1_500_000), + 'every batch fits the byte limit' + ); + + // The rating a master should have (V5): P2's estimate with P6's weight. + const rated = expectedMasterRatings({ + reviews: [ + review(11, 'l1', { rating_star: 5, criteria_ratings: { battery: 4 } }), + review(12, 'l1', { rating_exact: 4, rating_star: 4, criteria_ratings: { battery: 5 } }), + review(13, 'l1', { + rating_star: 2, + helpfulness_weight: 0.4, + criteria_ratings: { battery: 3, 'Bad Key': 5 }, + }), + // Not counted: excluded, no flag, an expert, weight 0, seed, a seed listing. + review(14, 'l1', { + review_flags: { include_in_rating: false, verdict: 'fake', flag_category: null }, + }), + review(15, 'l1', { review_flags: null }), + review(16, 'l1', { reviewer_type: 'expert' }), + review(17, 'l1', { helpfulness_weight: 0 }), + review(18, 'l1', { origin: 'seed' }), + review(19, 'lseed'), + // A NULL include counts. + review(20, 'l2', { + rating_star: 3, + review_flags: { include_in_rating: null, verdict: null, flag_category: null }, + }), + ], + listings, + values: new Map([[id(11), { weightCompleteness: 35 }]]), + }); + // M1: weights 1.25, 1, 0.4: (12.6 + 6.25 + 4 + 0.8) / 5.65 = 4.186. + assert.deepEqual(rated.get(M1), { + rating_score: 4.186, + rating_weight_sum: 2.65, + criteria_scores: { + // (12.6 + 5 + 5 + 1.2) / 5.65; three writers, shown 2.4. + battery: { score: 4.212, weight_sum: 2.65, writers: 3 }, + }, + }); + assert.deepEqual(rated.get(M2), { rating_score: 3.9, rating_weight_sum: 1, criteria_scores: {} }); + // D9's threshold on weights that sum to exactly 1.5 (0.7 + 0.35 + 0.45): floating point + // gives 1.4999999999999998 in this order and would drop the criterion. + const edge = expectedMasterRatings({ + reviews: [0.7, 0.35, 0.45].map((w, i) => + review(30 + i, 'l2', { helpfulness_weight: w, criteria_ratings: { battery: 4 } }) + ), + listings, + values: new Map(), + }); + assert.deepEqual(edge.get(M2)!.criteria_scores, { + battery: { score: 4.133, weight_sum: 1.5, writers: 3 }, + }); + assert.ok( + !masterRatingMatches( + { + id: M2, + rating_score: 4.134, + rating_weight_sum: 1.5, + criteria_scores: edge.get(M2)!.criteria_scores, + }, + edge.get(M2) + ), + 'one unit in the last place is stale, as V5 compares exactly' + ); + assert.ok( + masterRatingMatches( + { + id: M1, + rating_score: '4.186', + rating_weight_sum: '2.65', + criteria_scores: { battery: { score: 4.212, weight_sum: 2.65, writers: 3 } }, + }, + rated.get(M1) + ) + ); + assert.ok( + !masterRatingMatches( + { id: M1, rating_score: 4.186, rating_weight_sum: 2.65, criteria_scores: {} }, + rated.get(M1) + ), + 'a missing criterion is stale' + ); + assert.ok( + !masterRatingMatches( + { id: M2, rating_score: 3.95, rating_weight_sum: 1, criteria_scores: {} }, + rated.get(M2) + ) + ); + assert.ok( + masterRatingMatches( + { id: M2, rating_score: null, rating_weight_sum: 0, criteria_scores: {} }, + undefined + ), + 'a master with nothing counted has no rating' + ); + + // A deadlock is retried; a refusal stops the run. + let calls = 0; + const flaky = { + rpc: () => { + calls += 1; + return Promise.resolve( + calls === 1 + ? { data: null, error: { code: '40P01', message: 'deadlock detected' } } + : { data: 1, error: null } + ); + }, + } as unknown as RateSupabaseClient; + assert.equal(await storeBatch(flaky, many.slice(0, 1), async () => {}), 1); + assert.equal(calls, 2); + const refusing = { + rpc: () => + Promise.resolve({ + data: null, + error: { code: '22023', message: 'row 0 has values out of range' }, + }), + } as unknown as RateSupabaseClient; + await assert.rejects( + storeBatch(refusing, many.slice(0, 1), async () => {}), + /22023/ + ); + let deadlocks = 0; + const stuck = { + rpc: () => { + deadlocks += 1; + return Promise.resolve({ + data: null, + error: { code: '40P01', message: 'deadlock detected' }, + }); + }, + } as unknown as RateSupabaseClient; + await assert.rejects( + storeBatch(stuck, many.slice(0, 1), async () => {}), + /40P01/ + ); + assert.equal(deadlocks, 4, 'four attempts, then the run stops'); + + // The job against a fake client. + const fake = fakeSupabase({ + reviews, + review_points: points, + review_values: [], + products: listings, + master_products: masters.map(m => ({ + ...m, + rating_score: null, + rating_weight_sum: 0, + criteria_scores: {}, + })), + categories, + }); + // The masters as P2 and P6 rate them once the values are stored, except M2, + // which another transaction held during its refresh. + const after = expectedMasterRatings({ reviews, listings, values }); + fake.onStore = () => { + const m1 = fake.tables.master_products!.find(r => r.id === M1)!; + Object.assign(m1, after.get(M1)); + }; + + const dry = await runReviewValuesJob(fake.client); + assert.equal(dry.apply, false); + assert.equal(dry.reviews, 7); + assert.equal(dry.changed, 7); + assert.equal(fake.stores.length, 0, 'a dry run stores nothing'); + assert.deepEqual(dry.staleMasters, [M1, M2], 'the baseline differs before any value is stored'); + + const applied = await runReviewValuesJob(fake.client, { apply: true }); + assert.equal(applied.stored, 7); + assert.equal(fake.stores.length, 1); + assert.deepEqual( + fake.stores[0]!.map(r => r.id), + [1, 2, 3, 4, 5, 7, 8].map(id), + 'one batch, by master then review id' + ); + assert.deepEqual(applied.staleMasters, [M2], 'M2 is reported after a fresh read'); + assert.equal( + fake.tables.reviews!.find(r => r.id === id(2))!.value_tier, + values.get(id(2))!.tier, + 'the tier is public' + ); + + const again = await runReviewValuesJob(fake.client, { apply: true }); + assert.equal(again.changed, 0, 'a second run stores nothing'); + assert.equal(fake.stores.length, 1); + + // One master only. + const one = fakeSupabase({ + reviews, + review_points: points, + review_values: [], + products: listings, + master_products: masters.map(m => ({ + ...m, + rating_score: null, + rating_weight_sum: 0, + criteria_scores: {}, + })), + categories, + }); + const single = await runReviewValuesJob(one.client, { apply: true, masterId: M2 }); + assert.equal(single.reviews, 2); + assert.deepEqual( + one.stores[0]!.map(r => r.id), + [7, 8].map(id) + ); + await assert.rejects(runReviewValuesJob(one.client, { masterId: 'not-a-uuid' }), /UUID/); + + // A big master: 2500 reviews with 2 lines each, read past PostgREST's 1000-row pages, + // both for the whole table and for one master. + const crowd = Array.from({ length: 2500 }, (_, i) => + review(1000 + i, 'l2', { comment: 'Pin trâu, màn hình đẹp, chụp ảnh nét.' }) + ); + const crowdLines = crowd.flatMap(r => [ + { + id: `${r.id}-a`, + review_id: r.id, + kind: 'pro', + body: 'Pin dùng cả ngày', + criterion_key: null, + method: 'writer', + }, + { + id: `${r.id}-b`, + review_id: r.id, + kind: 'con', + body: 'Sạc hơi chậm', + criterion_key: null, + method: 'writer', + }, + ]); + for (const masterId of [null, M2]) { + const bigFake = fakeSupabase({ + reviews: crowd, + review_points: crowdLines, + review_values: [], + products: listings, + master_products: masters.map(m => ({ + ...m, + rating_score: null, + rating_weight_sum: 0, + criteria_scores: {}, + })), + categories, + }); + const big = await runReviewValuesJob(bigFake.client, { apply: true, masterId }); + assert.equal(big.reviews, 2500, `every review read (${masterId ?? 'all'})`); + assert.equal(bigFake.tables.review_values!.length, 2500); + assert.deepEqual( + bigFake.stores.map(b => b.length), + [500, 500, 500, 500, 500] + ); + assert.ok( + bigFake.stores.every(b => b.every(r => r.breakdown.gaps.includes('criteria'))), + 'the lines were read: no review misses its pros and cons' + ); + assert.ok( + bigFake.stores.flat().every(r => !r.breakdown.gaps.includes('pros_cons')), + 'every review has its lines' + ); + } + + console.log( + ' ✅ inputs, changed values, batches, deadlock retry, V5 check, dry run and apply verified' + ); +} + +type Row = Record; + +/** + * A tiny stand-in for the Supabase query builder (select, in, gt, order and + * limit, at most 1000 rows a read as PostgREST) and the writer RPC, which upserts rate.review_values and sets the + * public tier as rate.store_review_values does. + */ +function fakeSupabase(seed: Record) { + const tables: Record = Object.fromEntries( + Object.entries(seed).map(([name, rows]) => [name, rows.map(r => ({ ...(r as Row) }))]) + ); + const stores: ReviewValueWrite[][] = []; + const reads: string[] = []; + const state: { onStore: (() => void) | null } = { onStore: null }; + const copy = (row: Row): Row => ({ ...row }); + const query = (table: string) => { + let rows = [...(tables[table] ?? [])]; + const builder = { + select: () => builder, + order: (key: string) => { + rows.sort((a, b) => (String(a[key]) < String(b[key]) ? -1 : 1)); + return builder; + }, + in: (column: string, values: unknown[]) => { + rows = rows.filter(r => values.includes(r[column])); + return builder; + }, + gt: (column: string, value: string) => { + rows = rows.filter(r => String(r[column]) > value); + return builder; + }, + // PostgREST returns at most 1000 rows, whatever the limit. + limit: (n: number) => { + reads.push(table); + return Promise.resolve({ data: rows.slice(0, Math.min(n, 1000)).map(copy), error: null }); + }, + }; + return builder; + }; + const client = { + from: (table: string) => query(table), + rpc: (name: string, args: { p_values: ReviewValueWrite[] }) => { + assert.equal(name, 'store_review_values'); + stores.push(args.p_values); + let changed = 0; + for (const row of JSON.parse(JSON.stringify(args.p_values)) as ReviewValueWrite[]) { + const values = tables.review_values!; + const old = values.find(v => v.review_id === row.id); + if (old) { + old.value_score = row.score; + old.value_breakdown = row.breakdown; + } else { + values.push({ + review_id: row.id, + value_score: row.score, + value_breakdown: row.breakdown, + }); + } + changed += 1; + const target = tables.reviews!.find(r => r.id === row.id); + if (target) target.value_tier = row.breakdown.tier; + } + state.onStore?.(); + return Promise.resolve({ data: changed, error: null }); + }, + }; + return { + client: client as unknown as RateSupabaseClient, + tables, + stores, + reads, + set onStore(fn: () => void) { + state.onStore = fn; + }, + }; +} + +run().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/apps/importer/src/shopee/ai-review-analyzer.ts b/apps/importer/src/shopee/ai-review-analyzer.ts index e1bac693..1f750546 100644 --- a/apps/importer/src/shopee/ai-review-analyzer.ts +++ b/apps/importer/src/shopee/ai-review-analyzer.ts @@ -1,6 +1,6 @@ import crypto from 'node:crypto'; import { z, MAX_VALID_VND_PRICE, seedingRiskPenalty } from '@repo/shared-data'; -import { scopeLlmReviewSummary } from '@repo/shared-data/rules'; +import { reviewDepth, scopeLlmReviewSummary, type ReviewDepth } from '@repo/shared-data/rules'; import { resolveValidHttpBaseUrl, extractJsonObjectString, @@ -80,14 +80,8 @@ export type Independence = | 'undisclosed_paid' | 'conflict_of_interest'; -export type Depth = - | 'stars_only' - | 'platform_label' - | 'filler' - | 'low_effort' - | 'specific_short' - | 'detailed' - | 'expert_depth'; +/** Axis C, as `reviewDepth` in @repo/shared-data/rules classifies it. */ +export type Depth = ReviewDepth; export type SubjectKind = 'master_product' | 'listing' | 'brand' | 'merchant' | 'reviewer' | 'expert' | 'review'; @@ -680,16 +674,12 @@ export function isRatingOnlyComment(comment: string | null | undefined): boolean const SHIPPING_ASPECT_RE = /(?:giao\s*hàng|ship(?:per)?|vận\s*chuyển|đóng\s*gói|bọc\s*chống\s*sốc|hộp\s*(?:bị\s*)?(?:móp|rách|nát)|giao\s*(?:nhanh|chậm|trễ|lâu)|nhân\s*viên\s*giao|đơn\s*vị\s*vận\s*chuyển)/i; -// Whole words only (Unicode-aware), so "hư" in "như" or "pin" in "shopping" -// do not count as a product aspect (DOS/Rate#601). -const PRODUCT_ASPECT_RE = - /(? = {}; @@ -968,30 +954,13 @@ export function assessReview6Axes( independence = 'affiliate'; } - // 4. Axis C: Depth - let depth: Depth; - if (cleanText.length === 0) { - depth = 'stars_only'; - } else if (RATING_LABEL_RE.test(cleanText)) { - depth = 'platform_label'; - } else if (options.isCoinSpam) { - depth = 'filler'; - } else if ( - safeReview.reviewer_type === 'expert' || - TECHNICAL_MEASUREMENT_RE.test(comment) || - (comment.length >= 240 && productAspectCount >= 2) - ) { - depth = 'expert_depth'; - } else if ( - (comment.length >= 90 && (productAspectCount >= 1 || PRODUCT_ASPECT_RE.test(comment))) || - productAspectCount >= 2 - ) { - depth = 'detailed'; - } else if (productAspectCount >= 1 || PRODUCT_ASPECT_RE.test(comment)) { - depth = comment.length >= 30 ? 'specific_short' : 'low_effort'; - } else { - depth = 'low_effort'; - } + // 4. Axis C: Depth, the classifier the web and the review value job use + // (@repo/shared-data/rules, DOS/Rate#1583). Coin spam is filler. + const depth: Depth = reviewDepth({ + comment, + reviewerType: safeReview.reviewer_type, + flagCategory: options.isCoinSpam ? 'low_value_coin_spam' : null, + }); const hasRealMedia = Array.isArray(safeReview.images) && diff --git a/apps/importer/src/shopee/jobs/review-values.ts b/apps/importer/src/shopee/jobs/review-values.ts new file mode 100644 index 00000000..b1e688c1 --- /dev/null +++ b/apps/importer/src/shopee/jobs/review-values.ts @@ -0,0 +1,844 @@ +/** + * Review value score writer (DOS/Rate#1547, design #1531 section 6, D6 and D7) + * + * Usage: + * pnpm --filter @repo/crawlers exec tsx src/shopee/jobs/review-values.ts [--apply] [--master ] + * + * Reads every crawled and community review with its flag, its lines + * (rate.review_points), its listing and category, computes reviewValue + * (@repo/shared-data/rules) with the review's depth on its whole text + * (reviewDepth), and stores the values that changed through + * rate.store_review_values. Dry-run by default: it logs how many values it + * would store (counts only: the numbers are private, D7). `--apply` writes, as the rate_worker role: the job signs in as the + * worker's Supabase Auth user (RATE_WORKER_EMAIL and RATE_WORKER_PASSWORD, + * with the project's publishable key), whose access token carries that role. + * It may read the review tables and write values only through the writer, + * which accepts only a breakdown the shared module could produce. + * + * The inputs, as rate.reviews holds them today: + * - gated: the review does not count, as the product points job counts it + * (no flag, excluded, weight 0; an expert review Rate published unless the + * analyzer excluded it), or its depth is filler (6.1); + * - a photo: the review has an image; votes: upvotes and downvotes, as the + * review sort counts them; verified: the marketplace's purchase mark; + * - the writer's trust (axis D) is not stored yet, so it counts as unknown. + * + * Batches hold at most BATCH_ROWS rows and BATCH_BYTES bytes (the writer takes + * 1000 rows and 2 MB). The writer locks a batch's reviews in id order; another + * writer of rate.reviews may not, so a batch a deadlock aborts (40P01) is + * retried. The other side of such a deadlock is usually a crawl writing + * reviews: schedule the job away from the crawl windows. Run it from one + * place at a time (the importer's schedule). Rows are read by keyset pages, + * and every read is held in memory: a few hundred MB per hundred thousand + * reviews with their text and lines. + * + * `stored` counts the review_values rows the writer changed; a row whose + * only change is the public tier counts in `changed`, not in `stored`. + * + * After a run it checks every rated master as V5 of the verify file does + * (scripts/ops/rate-review-value-score-verify.sql in DOS-Me): the stored + * rating, weight sum and criteria scores against what P2's rating with the + * P6 weight gives. rate_worker cannot call rate.master_rating_extras, so the + * job computes it from the rows it read; a master that differs is read again + * on its own before it is reported, since a crawl may have moved it during + * the run. A master still stale was held by another transaction when its + * refresh ran (P2 skips a locked master); service_role refreshes it with + * rate.refresh_master_rating_extras. The CLI exits with code 2 when any is + * left. Masters are scoped by cluster_id: Items (M1+M2) are not in + * production; this check needs their scope once they are. + */ +import { pathToFileURL } from 'node:url'; +import dotenv from 'dotenv'; +import { + reviewDepth, + completenessWeightFactor, + reviewValue, + type ReviewValue, + type ReviewValuePoint, +} from '@repo/shared-data/rules'; +import { resolveBackfillSupabaseTarget } from '../paging.js'; +import { createImporterLogger } from '../logger.js'; +import { scopeSupabaseToRate, type RateSupabaseClient } from '../types.js'; +import { reviewPointWeight, type CategoryRow, type PointMasterRow } from './product-points.js'; + +const logger = createImporterLogger('ReviewValuesJob'); + +/** IDs per `in` filter, so a request URL stays short. */ +const IN_CHUNK = 150; +/** Rows per read; PostgREST returns at most 1000. */ +export const PAGE_ROWS = 1000; +/** Rows per writer call; the writer takes 1000. */ +export const BATCH_ROWS = 500; +/** Bytes per writer call; the writer takes 2 MB. */ +export const BATCH_BYTES = 1_500_000; +/** Attempts per batch when a deadlock (40P01) aborts it. */ +export const DEADLOCK_ATTEMPTS = 4; +/** Stale master ids a log line lists; the result has all of them. */ +const STALE_LOGGED = 50; + +type FlagRow = { + include_in_rating: boolean | null; + verdict: string | null; + flag_category: string | null; +}; + +export interface ValueReviewRow { + id: string; + product_id: string; + origin: string | null; + reviewer_type: string | null; + helpfulness_weight: number | string | null; + rating_star: number | null; + rating_exact: number | string | null; + comment: string | null; + verdict: string | null; + criteria_ratings: unknown; + images: string[] | null; + is_verified_buyer: boolean | null; + upvotes: number | null; + downvotes: number | null; + value_tier: string | null; + review_flags: FlagRow | FlagRow[] | null; +} + +export interface ValuePointRow { + id?: string; + review_id: string; + kind: string; + criterion_key: string | null; + body: string; + method: string; +} + +export interface ValueListingRow { + id: string; + cluster_id: string | null; + origin: string | null; +} + +export interface StoredValueRow { + review_id: string; + value_score: number | string; + value_breakdown: unknown; +} + +export interface MasterRatingRow { + id: string; + rating_score: number | string | null; + rating_weight_sum: number | string | null; + criteria_scores: unknown; +} + +/** One row of rate.store_review_values. */ +export interface ReviewValueWrite { + id: string; + score: number; + breakdown: ReviewValue; +} + +const REVIEW_COLUMNS = + 'id, product_id, origin, reviewer_type, helpfulness_weight, rating_star, rating_exact, ' + + 'comment, verdict, criteria_ratings, images, is_verified_buyer, upvotes, downvotes, ' + + 'value_tier, review_flags(include_in_rating, verdict, flag_category)'; + +/** Reviews the job scores: crawled or community, never seed or test data. */ +const SCORED_REVIEW_ORIGINS = ['crawl', 'community']; + +const flagOf = (review: ValueReviewRow): FlagRow | null => + Array.isArray(review.review_flags) ? (review.review_flags[0] ?? null) : review.review_flags; + +const count = (value: unknown): number => { + const n = Number(value ?? 0); + return Number.isFinite(n) && n > 0 ? n : 0; +}; + +/** The shared module's input for one review. */ +export function reviewValueInput( + review: ValueReviewRow, + lines: readonly ValuePointRow[], + categorySlug: string | null +): Parameters[0] { + const flag = flagOf(review); + const points: ReviewValuePoint[] = lines + .filter(l => l.kind === 'pro' || l.kind === 'con') + .map(l => ({ + kind: l.kind as 'pro' | 'con', + criterionKey: l.criterion_key, + body: l.body, + method: l.method === 'ai' ? 'ai' : 'writer', + })); + const depth = reviewDepth({ + comment: review.comment, + verdict: review.verdict, + points, + reviewerType: review.reviewer_type, + flagCategory: flag?.flag_category ?? null, + }); + const weight = reviewPointWeight({ + id: review.id, + product_id: review.product_id, + origin: review.origin, + reviewer_type: review.reviewer_type, + helpfulness_weight: review.helpfulness_weight, + created_at: null, + platform_created_at: null, + review_flags: flag, + }); + return { + categorySlug, + gated: weight === 0 || depth === 'filler', + depth, + criteriaRatings: review.criteria_ratings, + verdict: review.verdict, + points, + hasRealPhoto: Array.isArray(review.images) && review.images.length > 0, + // A marketplace's mark only: is_verified_buyer defaults to true, and community + // reviews written before 20261018100000 took that default without a purchase. + verifiedPurchase: review.origin === 'crawl' && review.is_verified_buyer === true, + helpfulVotes: count(review.upvotes), + unhelpfulVotes: count(review.downvotes), + authorTrust: null, + }; +} + +/** The category slug of each listing's master, or null. */ +function categoryByListing(input: { + listings: readonly ValueListingRow[]; + masters: readonly PointMasterRow[]; + categories: readonly CategoryRow[]; +}): Map { + const slugByCategory = new Map(input.categories.map(c => [c.id, c.slug])); + const slugByMaster = new Map( + input.masters.map(m => [ + m.id, + m.category_id ? (slugByCategory.get(m.category_id) ?? null) : null, + ]) + ); + return new Map( + input.listings.map(l => [l.id, l.cluster_id ? (slugByMaster.get(l.cluster_id) ?? null) : null]) + ); +} + +/** The value of every review, by review id. */ +export function buildReviewValues(input: { + reviews: readonly ValueReviewRow[]; + points: readonly ValuePointRow[]; + listings: readonly ValueListingRow[]; + masters: readonly PointMasterRow[]; + categories: readonly CategoryRow[]; +}): Map { + const linesByReview = new Map(); + for (const line of input.points) { + const lines = linesByReview.get(line.review_id) ?? []; + lines.push(line); + linesByReview.set(line.review_id, lines); + } + const slugOf = categoryByListing(input); + const out = new Map(); + for (const review of input.reviews) { + if (!SCORED_REVIEW_ORIGINS.includes(review.origin ?? '')) continue; + out.set( + review.id, + reviewValue( + reviewValueInput( + review, + linesByReview.get(review.id) ?? [], + slugOf.get(review.product_id) ?? null + ) + ) + ); + } + return out; +} + +/** JSON with sorted keys, so a stored jsonb and a computed object compare. */ +function canonical(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + if (value && typeof value === 'object') { + const entries = Object.entries(value as Record).sort(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0 + ); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonical(v)}`).join(',')}}`; + } + return JSON.stringify(value ?? null); +} + +/** + * The rows to store: reviews whose stored number, breakdown or public tier + * differs from the computed value. They are ordered by master, then review + * id: each writer call refreshes the masters of its batch, so a master's + * reviews in one or two batches refresh it once or twice, not once per + * batch. The writer still locks each batch in review id order. + */ +export function changedValues( + values: ReadonlyMap, + stored: readonly StoredValueRow[], + tierById: ReadonlyMap, + masterById: ReadonlyMap = new Map() +): ReviewValueWrite[] { + const storedById = new Map(stored.map(row => [row.review_id, row])); + const out: ReviewValueWrite[] = []; + for (const [id, value] of values) { + const row = storedById.get(id); + const same = + row !== undefined && + Number(row.value_score) === value.score && + canonical(row.value_breakdown) === canonical(value) && + (tierById.get(id) ?? null) === value.tier; + if (!same) out.push({ id, score: value.score, breakdown: value }); + } + const order = (row: ReviewValueWrite) => `${masterById.get(row.id) ?? '~'}|${row.id}`; + return out.sort((a, b) => (order(a) < order(b) ? -1 : order(a) > order(b) ? 1 : 0)); +} + +/** Writer calls of at most BATCH_ROWS rows and BATCH_BYTES bytes each. */ +export function batchValues(rows: readonly ReviewValueWrite[]): ReviewValueWrite[][] { + const out: ReviewValueWrite[][] = []; + let batch: ReviewValueWrite[] = []; + let bytes = 2; + for (const row of rows) { + const size = Buffer.byteLength(JSON.stringify(row), 'utf8') + 1; + if (batch.length > 0 && (batch.length >= BATCH_ROWS || bytes + size > BATCH_BYTES)) { + out.push(batch); + batch = []; + bytes = 2; + } + batch.push(row); + bytes += size; + } + if (batch.length > 0) out.push(batch); + return out; +} + +// The rating is computed in exact integers, as PostgreSQL's numeric does: a +// weight in millionths (helpfulness_weight has 2 decimals, the factor 4), a +// star or criterion value in ten-thousandths. Floating point would flip D9's +// `>= 1.5` threshold on sums like 0.7 + 0.35 + 0.45. +const W_UNIT = 1_000_000n; +const V_UNIT = 10_000n; +const PRIOR_SUM = 126n * W_UNIT * V_UNIT; // k x mu = 3 x 4.2 = 12.6, x 10 +const PRIOR_WEIGHT = 3n * W_UNIT; + +const toUnits = (value: number, unit: bigint): bigint => BigInt(Math.round(value * Number(unit))); + +/** round(num / den, digits) half away from zero, for num, den >= 0. */ +const divRound = (num: bigint, den: bigint, digits: number): number => { + const scale = 10n ** BigInt(digits); + return Number((2n * num * scale + den) / (2n * den)) / Number(scale); +}; + +/** (k x mu + sum(w x v)) / (k + sum(w)) at 3 decimals; sumWV carries PRIOR_SUM's /10. */ +const bayes = (sumW: bigint, sumWV: bigint): number => + divRound(PRIOR_SUM + 10n * sumWV, 10n * V_UNIT * (PRIOR_WEIGHT + sumW), 3); + +const weightSum = (sumW: bigint): number => divRound(sumW, W_UNIT, 2); + +/** What rate.master_rating_extras gives for a master, from the rows read. */ +export interface MasterRatingExpected { + rating_score: number | null; + rating_weight_sum: number; + criteria_scores: Record; +} + +const CRITERION_KEY = /^[a-z_]{2,32}$/; + +/** + * P2's rating with P6's weight, per master (cluster_id scope): user reviews + * of crawled listings, crawl or community, with a flag that does not exclude + * them and a weight above 0, each weighing LEAST(3, weight x factor). + */ +export function expectedMasterRatings(input: { + reviews: readonly ValueReviewRow[]; + listings: readonly ValueListingRow[]; + values: ReadonlyMap; +}): Map { + const masterByListing = new Map( + input.listings + .filter(l => l.origin === 'crawl' && typeof l.cluster_id === 'string' && l.cluster_id) + .map(l => [l.id, l.cluster_id as string]) + ); + type Eligible = { w: bigint; star: number; criteria: unknown }; + const byMaster = new Map(); + for (const review of input.reviews) { + const masterId = masterByListing.get(review.product_id); + if (!masterId) continue; + const flag = flagOf(review); + const weight = Number(review.helpfulness_weight ?? 0); + if ( + !SCORED_REVIEW_ORIGINS.includes(review.origin ?? '') || + review.reviewer_type !== 'user' || + flag === null || + flag.include_in_rating === false || + !(weight > 0) + ) { + continue; + } + const exact = review.rating_exact == null ? null : Number(review.rating_exact); + const list = byMaster.get(masterId) ?? []; + const factor = completenessWeightFactor(input.values.get(review.id)?.weightCompleteness); + const w = toUnits(weight, 100n) * toUnits(factor, V_UNIT); + list.push({ + // LEAST(3, helpfulness_weight x factor), as reviewRatingWeight, in millionths. + w: w < 3n * W_UNIT ? w : 3n * W_UNIT, + star: exact ?? Number(review.rating_star), + criteria: review.criteria_ratings, + }); + byMaster.set(masterId, list); + } + const out = new Map(); + for (const [masterId, eligible] of byMaster) { + const rated = eligible.filter(e => e.star >= 1 && e.star <= 5); + const sumW = rated.reduce((s, e) => s + e.w, 0n); + const sumWS = rated.reduce((s, e) => s + e.w * toUnits(e.star, V_UNIT), 0n); + const perKey = new Map(); + for (const e of eligible) { + if (!e.criteria || typeof e.criteria !== 'object' || Array.isArray(e.criteria)) continue; + for (const [key, value] of Object.entries(e.criteria as Record)) { + if (!CRITERION_KEY.test(key) || typeof value !== 'number' || value < 1 || value > 5) { + continue; + } + const acc = perKey.get(key) ?? { writers: 0, w: 0n, shown: 0n, ws: 0n }; + acc.writers += 1; + acc.w += e.w; + acc.shown += e.w < W_UNIT ? e.w : W_UNIT; + acc.ws += e.w * toUnits(value, V_UNIT); + perKey.set(key, acc); + } + } + const criteria: MasterRatingExpected['criteria_scores'] = {}; + for (const [key, acc] of perKey) { + if (acc.writers < 3 || 2n * acc.shown < 3n * W_UNIT) continue; + criteria[key] = { + score: bayes(acc.w, acc.ws), + weight_sum: weightSum(acc.w), + writers: acc.writers, + }; + } + out.set(masterId, { + rating_score: sumW > 0n ? bayes(sumW, sumWS) : null, + rating_weight_sum: weightSum(sumW), + criteria_scores: criteria, + }); + } + return out; +} + +/** Equal at `digits` decimals, as V5 compares the stored numeric exactly. */ +const close = (a: unknown, b: number | null, digits: number): boolean => { + if (a === null || a === undefined || b === null) return (a ?? null) === b; + const scale = 10 ** digits; + return Math.round(Number(a) * scale) === Math.round(b * scale); +}; + +/** Whether a stored master equals what the rating gives (V5 of the verify file). */ +export function masterRatingMatches( + row: MasterRatingRow, + expected: MasterRatingExpected | undefined +): boolean { + const want: MasterRatingExpected = expected ?? { + rating_score: null, + rating_weight_sum: 0, + criteria_scores: {}, + }; + if (!close(row.rating_score, want.rating_score, 3)) return false; + if (!close(row.rating_weight_sum ?? 0, want.rating_weight_sum, 2)) return false; + const stored = + row.criteria_scores && typeof row.criteria_scores === 'object' + ? (row.criteria_scores as Record< + string, + { score?: unknown; weight_sum?: unknown; writers?: unknown } + >) + : {}; + const keys = Object.keys(want.criteria_scores).sort(); + if (canonical(Object.keys(stored).sort()) !== canonical(keys)) return false; + return keys.every(key => { + const s = stored[key] ?? {}; + const w = want.criteria_scores[key]!; + return ( + close(s.score, w.score, 3) && + close(s.weight_sum, w.weight_sum, 2) && + Number(s.writers) === w.writers + ); + }); +} + +function chunks(items: readonly T[], size = IN_CHUNK): T[][] { + const out: T[][] = []; + for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size)); + return out; +} + +type Filter = (query: ReturnType['select']>) => unknown; + +/** + * Every row of a table, or of a filter on it, in pages of PAGE_ROWS keyed on + * `key` (keyset, not OFFSET): each page costs the same however deep it is, + * a row written during the read never shifts the next page, and there is no + * page cap. `columns` must include `key`. + */ +export async function fetchByKey( + supabase: RateSupabaseClient, + table: string, + columns: string, + key: string, + filter: Filter = query => query +): Promise { + const rows: T[] = []; + let last: string | null = null; + for (;;) { + let query = filter(supabase.from(table).select(columns)) as ReturnType< + ReturnType['select'] + >; + if (last !== null) query = query.gt(key, last); + const { data, error } = await query.order(key).limit(PAGE_ROWS); + if (error) throw new Error(`${table}: ${error.message}`); + const page = (data ?? []) as unknown as T[]; + rows.push(...page); + if (page.length < PAGE_ROWS) return rows; + last = String((page[page.length - 1] as Record)[key]); + } +} + +/** The rows whose `column` is one of `ids`, every page of each chunk. */ +async function selectIn( + supabase: RateSupabaseClient, + table: string, + columns: string, + column: string, + ids: readonly string[], + key = 'id' +): Promise { + const rows: T[] = []; + for (const part of chunks([...new Set(ids)])) { + rows.push( + ...(await fetchByKey(supabase, table, columns, key, query => query.in(column, part))) + ); + } + return rows; +} + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +/** Stores one batch, retrying a deadlock; returns the rows the writer changed. */ +export async function storeBatch( + supabase: RateSupabaseClient, + batch: readonly ReviewValueWrite[], + wait: (ms: number) => Promise = sleep +): Promise { + for (let attempt = 1; ; attempt++) { + const { data, error } = await supabase.rpc('store_review_values', { p_values: batch }); + if (!error) return Number(data ?? 0); + if (error.code === '40P01' && attempt < DEADLOCK_ATTEMPTS) { + logger.warn( + 'review_values.deadlock', + `Batch of ${batch.length} hit a deadlock; retrying (${attempt}/${DEADLOCK_ATTEMPTS - 1}).`, + { attempt } + ); + await wait(250 * 2 ** (attempt - 1)); + continue; + } + // 22023: the writer refused a row the module produced, so the module and + // the database disagree (a REVIEW_VALUE_VERSION change needs DOS-Me first). + throw new Error(`store_review_values (${error.code ?? 'no code'}): ${error.message}`); + } +} + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** The rows a master's rating reads, read again on their own. */ +async function loadMasterRating( + supabase: RateSupabaseClient, + masterIds: readonly string[] +): Promise<{ + rows: MasterRatingRow[]; + expected: Map; +}> { + const listings = await selectIn( + supabase, + 'products', + 'id, cluster_id, origin', + 'cluster_id', + masterIds + ); + const reviews = await selectIn( + supabase, + 'reviews', + REVIEW_COLUMNS, + 'product_id', + listings.map(l => l.id) + ); + const stored = await selectIn( + supabase, + 'review_values', + 'review_id, value_score, value_breakdown', + 'review_id', + reviews.map(r => r.id), + 'review_id' + ); + const values = new Map( + stored.map(row => [ + row.review_id, + { + weightCompleteness: (row.value_breakdown as { weightCompleteness?: unknown } | null) + ?.weightCompleteness, + }, + ]) + ); + const rows = await selectIn( + supabase, + 'master_products', + 'id, rating_score, rating_weight_sum, criteria_scores', + 'id', + masterIds + ); + return { rows, expected: expectedMasterRatings({ reviews, listings, values }) }; +} + +export interface ReviewValuesResult { + apply: boolean; + reviews: number; + changed: number; + stored: number; + batches: number; + masters: number; + staleMasters: string[]; +} + +export async function runReviewValuesJob( + supabase: RateSupabaseClient, + options: { + apply?: boolean; + masterId?: string | null; + wait?: (ms: number) => Promise; + } = {} +): Promise { + const apply = options.apply === true; + const only = options.masterId ?? null; + if (only !== null && !UUID.test(only)) throw new Error('--master must be a UUID.'); + + // One master: its listings first, then their reviews. Otherwise every scored review. + const scopedListings = + only === null + ? null + : await selectIn( + supabase, + 'products', + 'id, cluster_id, origin', + 'cluster_id', + [only] + ); + const reviews = + scopedListings === null + ? await fetchByKey(supabase, 'reviews', REVIEW_COLUMNS, 'id', query => + query.in('origin', SCORED_REVIEW_ORIGINS) + ) + : ( + await selectIn( + supabase, + 'reviews', + REVIEW_COLUMNS, + 'product_id', + scopedListings.map(l => l.id) + ) + ).filter(r => SCORED_REVIEW_ORIGINS.includes(r.origin ?? '')); + const reviewIds = reviews.map(r => r.id); + const pointColumns = 'id, review_id, kind, criterion_key, body, method'; + const points = + only === null + ? await fetchByKey(supabase, 'review_points', pointColumns, 'id') + : await selectIn( + supabase, + 'review_points', + pointColumns, + 'review_id', + reviewIds + ); + const valueColumns = 'review_id, value_score, value_breakdown'; + const stored = + only === null + ? await fetchByKey(supabase, 'review_values', valueColumns, 'review_id') + : await selectIn( + supabase, + 'review_values', + valueColumns, + 'review_id', + reviewIds, + 'review_id' + ); + const listings = + scopedListings ?? + (await selectIn( + supabase, + 'products', + 'id, cluster_id, origin', + 'id', + reviews.map(r => r.product_id) + )); + const masterIds = [ + ...new Set(listings.map(l => l.cluster_id).filter((id): id is string => !!id)), + ]; + const masters = await selectIn( + supabase, + 'master_products', + 'id, category_id', + 'id', + masterIds + ); + const categories = await fetchByKey(supabase, 'categories', 'id, slug', 'id'); + + const values = buildReviewValues({ reviews, points, listings, masters, categories }); + const masterOfListing = new Map(listings.map(l => [l.id, l.cluster_id])); + const changed = changedValues( + values, + stored, + new Map(reviews.map(r => [r.id, r.value_tier])), + new Map(reviews.map(r => [r.id, masterOfListing.get(r.product_id) ?? null])) + ); + const batches = batchValues(changed); + logger.info( + 'review_values.plan', + `${apply ? 'Storing' : 'Would store'} ${changed.length} of ${values.size} review values in ${batches.length} batches.`, + { apply, reviews: values.size, changed: changed.length, batches: batches.length } + ); + + let written = 0; + if (apply) { + for (const [index, batch] of batches.entries()) { + written += await storeBatch(supabase, batch, options.wait); + logger.info( + 'review_values.batch', + `Stored batch ${index + 1}/${batches.length} (${batch.length} rows).`, + { batch: index + 1, rows: batch.length } + ); + } + } + + // V5: in a dry run, the values stored now; after --apply, the values just stored. + const effective = new Map(); + for (const row of stored) { + effective.set(row.review_id, { + weightCompleteness: (row.value_breakdown as { weightCompleteness?: unknown } | null) + ?.weightCompleteness, + }); + } + if (apply) for (const row of changed) effective.set(row.id, row.breakdown); + const expected = expectedMasterRatings({ reviews, listings, values: effective }); + const ratedMasters = [...new Set([...expected.keys(), ...masterIds])]; + const current = await selectIn( + supabase, + 'master_products', + 'id, rating_score, rating_weight_sum, criteria_scores', + 'id', + ratedMasters + ); + const suspects = current.filter(row => !masterRatingMatches(row, expected.get(row.id))); + let staleMasters: string[] = []; + if (suspects.length > 0) { + const again = await loadMasterRating( + supabase, + suspects.map(row => row.id) + ); + staleMasters = again.rows + .filter(row => !masterRatingMatches(row, again.expected.get(row.id))) + .map(row => row.id) + .sort(); + } + if (staleMasters.length > 0) { + logger.warn( + 'review_values.stale_masters', + `${staleMasters.length} masters differ from their computed rating (V5); service_role refreshes them with rate.refresh_master_rating_extras.`, + { + masters: staleMasters.slice(0, STALE_LOGGED), + more: Math.max(0, staleMasters.length - STALE_LOGGED), + } + ); + } + + logger.info( + 'review_values.done', + `${apply ? 'Stored' : 'Dry run:'} ${apply ? written : changed.length} review values; ${ratedMasters.length} masters checked, ${staleMasters.length} stale.`, + { + apply, + reviews: values.size, + changed: changed.length, + stored: written, + masters: ratedMasters.length, + stale: staleMasters.length, + } + ); + return { + apply, + reviews: values.size, + changed: changed.length, + stored: written, + batches: batches.length, + masters: ratedMasters.length, + staleMasters, + }; +} + +const isDirectExecution = + typeof process !== 'undefined' && + Array.isArray(process.argv) && + Boolean(process.argv[1]) && + import.meta.url === pathToFileURL(process.argv[1]!).href; + +if (isDirectExecution) { + dotenv.config({ quiet: true }); + dotenv.config({ path: '../../.env', quiet: true }); + (async () => { + const { createClient } = await import('@supabase/supabase-js'); + const rawUrl = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL || ''; + // The publishable key plus the worker user's sign-in: its access token + // carries the rate_worker role, which reads the review tables and writes + // values only through rate.store_review_values. + const key = ( + process.env.SUPABASE_PUBLISHABLE_KEY || + process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY || + '' + ).trim(); + const email = (process.env.RATE_WORKER_EMAIL || '').trim(); + const password = process.env.RATE_WORKER_PASSWORD || ''; + if (!rawUrl.trim() || !key || !email || !password) { + throw new Error( + 'Missing SUPABASE_URL, SUPABASE_PUBLISHABLE_KEY, RATE_WORKER_EMAIL or RATE_WORKER_PASSWORD in environment.' + ); + } + const { url, host } = resolveBackfillSupabaseTarget(rawUrl); + const client = createClient(url, key, { auth: { persistSession: false } }); + const { data: auth, error: authError } = await client.auth.signInWithPassword({ + email, + password, + }); + if (authError || !auth.session) { + throw new Error( + `Sign-in as the rate_worker user failed: ${authError?.message ?? 'no session'}` + ); + } + const role = (auth.user?.role ?? '').trim(); + if (role !== 'rate_worker') { + throw new Error(`The signed-in user has role '${role}', not rate_worker; refusing to run.`); + } + const supabase = scopeSupabaseToRate(client); + const apply = process.argv.includes('--apply'); + const at = process.argv.indexOf('--master'); + const masterId = at >= 0 ? (process.argv[at + 1] ?? '') : null; + // The host is not secret; it shows which project the job reads. + logger.info('review_values.target', `Target: ${host} (${apply ? 'apply' : 'dry run'})`, { + host, + }); + try { + const result = await runReviewValuesJob(supabase, { apply, masterId }); + if (result.staleMasters.length > 0) process.exitCode = 2; + } finally { + // One run, one session: do not leave its refresh token behind. + await client.auth.signOut().catch(() => undefined); + } + })().catch((err: unknown) => { + logger.error( + 'review_values.fatal', + `Review values job failed: ${err instanceof Error ? err.message : String(err)}` + ); + process.exit(1); + }); +} diff --git a/apps/importer/src/shopee/test-review-analyzer.ts b/apps/importer/src/shopee/test-review-analyzer.ts index 50639024..3dd22478 100644 --- a/apps/importer/src/shopee/test-review-analyzer.ts +++ b/apps/importer/src/shopee/test-review-analyzer.ts @@ -4,7 +4,7 @@ */ import http from 'node:http'; import assert from 'node:assert/strict'; -import { LLM_REVIEW_SUMMARY_PREFIX } from '@repo/shared-data/rules'; +import { LLM_REVIEW_SUMMARY_PREFIX, reviewDepth } from '@repo/shared-data/rules'; import { readFileSync } from 'node:fs'; import { AiReviewAnalyzer, @@ -2934,6 +2934,40 @@ async function run() { ); } + // The analyzer's axis C is the shared reviewDepth, as on the web and in the + // review value job (DOS/Rate#1583). + { + for (const comment of [ + '', + 'Hài lòng', + 'Tốt', + 'Pin trâu', + 'Pin trâu, camera đẹp', + 'Pin dùng được cả ngày, màn hình đẹp.', + 'Pin 5000mAh dùng cả ngày', + 'Pin dùng được cả ngày với nhu cầu lướt web, màn hình sáng rõ ngoài trời. '.repeat(2), + Array(40).fill('pin ok').join(' '), + ]) { + assert.equal( + assessReview6Axes({ id: 'd', rating_star: 5, comment }).depth, + reviewDepth({ comment }), + 'depth of ' + JSON.stringify(comment.slice(0, 40)) + ); + } + assert.equal( + assessReview6Axes({ id: 'd', rating_star: 5, comment: 'Pin trâu' }, { isCoinSpam: true }) + .depth, + 'filler' + ); + assert.equal( + assessReview6Axes({ id: 'd', rating_star: 5, comment: 'ok', reviewer_type: 'expert' }).depth, + 'expert_depth' + ); + console.log( + '\u2705 [Axis C] the analyzer classifies depth with the shared reviewDepth (#1583)' + ); + } + console.log('\n🎉 ALL AI REVIEW ANALYZER TESTS PASSED!'); } diff --git a/apps/web/locales/cs.json b/apps/web/locales/cs.json index ebe4347b..14ef6d74 100644 --- a/apps/web/locales/cs.json +++ b/apps/web/locales/cs.json @@ -546,6 +546,18 @@ "noMatchingReviews": "Žádné recenze neodpovídají zvolenému filtru.", "roleExpertBadge": "Expert Rate", "roleBuyerBadge": "Kupující", + "valueTierTitle": "Úroveň kvality recenze", + "valueTierComplete": "Úplná", + "valueTierExcellent": "Výborná", + "myReviewValue": "Hodnota vaší recenze (vidíte jen vy): {score}/100", + "myReviewValueGaps": "Doplňte: {gaps}", + "valueGapDetail": "podrobnosti", + "valueGapCriteria": "všechna kritéria", + "valueGapVerdict": "závěr", + "valueGapProsCons": "klady a zápory", + "valueGapCriterionPoints": "klady a zápory u kritérií", + "valueGapPhoto": "skutečnou fotku", + "valueGapVerified": "ověřený nákup", "routedToMerchantBadge": "Přesměrováno do skóre prodejce", "sponsorshipBadgeSponsoredBalanced": "Sponzorováno", "sponsorshipBadgeSponsoredPromo": "Reklama", diff --git a/apps/web/locales/en.json b/apps/web/locales/en.json index 887856d5..18332415 100644 --- a/apps/web/locales/en.json +++ b/apps/web/locales/en.json @@ -546,6 +546,18 @@ "noMatchingReviews": "No reviews match the current filter criteria.", "roleExpertBadge": "Rate expert", "roleBuyerBadge": "Buyer", + "valueTierTitle": "Review quality tier", + "valueTierComplete": "Complete", + "valueTierExcellent": "Excellent", + "myReviewValue": "Your review value (only you see it): {score}/100", + "myReviewValueGaps": "Add: {gaps}", + "valueGapDetail": "detail", + "valueGapCriteria": "all criteria", + "valueGapVerdict": "a verdict", + "valueGapProsCons": "pros and cons", + "valueGapCriterionPoints": "pros and cons per criterion", + "valueGapPhoto": "a real photo", + "valueGapVerified": "a verified purchase", "routedToMerchantBadge": "Routed to merchant score", "sponsorshipBadgeSponsoredBalanced": "Sponsored", "sponsorshipBadgeSponsoredPromo": "Advertising", diff --git a/apps/web/locales/vi.json b/apps/web/locales/vi.json index 77185f1b..7d35d1d3 100644 --- a/apps/web/locales/vi.json +++ b/apps/web/locales/vi.json @@ -546,6 +546,18 @@ "noMatchingReviews": "Không có đánh giá nào khớp với bộ lọc hiện tại.", "roleExpertBadge": "Chuyên gia Rate", "roleBuyerBadge": "Người mua", + "valueTierTitle": "Bậc chất lượng của đánh giá", + "valueTierComplete": "Đầy đủ", + "valueTierExcellent": "Xuất sắc", + "myReviewValue": "Điểm giá trị bài của bạn (chỉ bạn thấy): {score}/100", + "myReviewValueGaps": "Thêm: {gaps}", + "valueGapDetail": "chi tiết", + "valueGapCriteria": "chấm đủ tiêu chí", + "valueGapVerdict": "kết luận", + "valueGapProsCons": "cả ưu lẫn nhược", + "valueGapCriterionPoints": "ưu nhược theo tiêu chí", + "valueGapPhoto": "ảnh thật", + "valueGapVerified": "xác nhận đã mua", "routedToMerchantBadge": "Chuyển sang điểm người bán", "sponsorshipBadgeSponsoredBalanced": "Có tài trợ", "sponsorshipBadgeSponsoredPromo": "Quảng cáo", diff --git a/apps/web/package.json b/apps/web/package.json index f6cb17c3..c5ce3b27 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -8,7 +8,7 @@ "lint": "eslint . --max-warnings 0", "preview": "next build && next start", "start": "next start", - "test": "pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/core-rate-architecture.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-model.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/write-paths.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/spec-scope.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/item-fields.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/logger.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-form-v2.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/concurrency.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-profile-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-review-risk.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/watchlist-sizes.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-compare.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-variant-links.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/compare-params.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/catalog-listing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/discount-verdict-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-detail-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-competitors.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/analyzer-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sold-count.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/content-lang.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/image-url.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/client-messages.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/seo.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/offer-plausibility.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-market-stats.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sitemap-lastmod.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-badge-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-criteria.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/rating-score.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/rating-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/focus-trap.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/display-truth.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/spec-template.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-spec-rows.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sale-state.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/csp.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/device-images.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/render-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/model-match.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/developers-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/plural-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/locale-format.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/paged-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/server-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/hero-products.test.ts", + "test": "pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/core-rate-architecture.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-model.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/write-paths.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/spec-scope.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/item-fields.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/logger.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-form-v2.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-value-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/concurrency.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-profile-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-review-risk.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/watchlist-sizes.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-compare.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-variant-links.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/compare-params.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/catalog-listing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/discount-verdict-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-detail-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-competitors.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/analyzer-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sold-count.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/content-lang.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/image-url.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/client-messages.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/seo.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/offer-plausibility.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-market-stats.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sitemap-lastmod.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-badge-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-criteria.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/trust-depth-v2.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/rating-score.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/rating-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/focus-trap.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/display-truth.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/spec-template.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-spec-rows.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sale-state.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/csp.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/device-images.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/render-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/model-match.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/developers-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/plural-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/locale-format.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/paged-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/server-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/hero-products.test.ts", "typecheck": "tsc --noEmit" }, "browserslist": [ diff --git a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts index 972362dc..8524c91d 100644 --- a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts +++ b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts @@ -394,7 +394,7 @@ async function testReviewsV2() { value: 3.5, }, verdict: 'Hợp người chụp ảnh nhiều, không hợp người chơi game nặng.', - pros: ['Pin dùng trọn ngày'], + pros: ['Pin dùng được 8 tiếng mỗi ngày'], cons: ['Giá hơi cao so với cấu hình'], }; rpcResponder = () => ({ status: 200, json: { id: 'stored' } }); @@ -561,6 +561,12 @@ async function testReviewsV2() { stored.review.signals.criteria_count, Object.keys(v2Review.criteriaRatings).length ); + // The new card's trust reads the verdict and the pros and cons, not the + // empty comment alone (DOS/Rate#1583). + assert.ok(stored.review.trust_assessment, 'the card carries its trust'); + // Only the pro measures the battery, so expert depth proves the lines + // reached the classifier. + assert.equal(stored.review.trust_assessment.depth, 'expert_depth'); const [v2Rpc] = rpcCalls('submit_community_review_v2'); assert.ok(v2Rpc, 'the v2 RPC must be called'); // v1 is never called on the v2 path. @@ -732,6 +738,97 @@ async function testReviewsByIdPublishedOnly() { ); } +async function testMyReviewValues() { + const { POST } = await import('../reviews/my-values/route'); + const MINE = 'aaaaaaaa-1111-4222-8333-444444444444'; + const OTHER = 'aaaaaaaa-1111-4222-8333-555555555555'; + const breakdown = { + version: 1, + gated: false, + score: 62, + tier: 'complete', + gaps: ['photo', 'not_a_gap', 'verified', 'photo'], + }; + + // Nobody signed in: no call to the database. + calls = []; + let res = await POST(post('/api/v1/rate/reviews/my-values', { ids: [MINE] })); + assert.equal(res.status, 401); + assert.equal(rpcCalls('my_review_values').length, 0); + + // Bad ids are refused before the database. + for (const ids of [ + 'x', + [MINE, 'not-a-uuid'], + Array.from( + { length: 101 }, + (_, i) => `aaaaaaaa-1111-4222-8333-${String(i).padStart(12, '0')}` + ), + ]) { + calls = []; + res = await POST( + post('/api/v1/rate/reviews/my-values', { ids }, GOOD_TOKEN) + ); + assert.equal(res.status, 400); + assert.equal(rpcCalls('my_review_values').length, 0); + } + + // The writer's call goes out with their own token; the answer keeps only + // asked ids with a valid score, the known gaps, and is never cached. + calls = []; + rpcResponder = (name) => + name === 'my_review_values' + ? { + status: 200, + json: [ + { review_id: MINE, value_score: 61.6, value_breakdown: breakdown }, + { + review_id: 'aaaaaaaa-1111-4222-8333-666666666666', + value_score: 90, + value_breakdown: breakdown, + }, + { review_id: OTHER, value_score: 'NaN', value_breakdown: null }, + ], + } + : { status: 500, json: { message: 'unexpected rpc' } }; + res = await POST( + post( + '/api/v1/rate/reviews/my-values', + { ids: [MINE, MINE.toUpperCase(), OTHER] }, + GOOD_TOKEN + ) + ); + assert.equal(res.status, 200); + assert.match(res.headers.get('cache-control') ?? '', /no-store/); + assert.deepEqual(await res.json(), { + ok: true, + values: [{ id: MINE, score: 62, gaps: ['photo', 'verified'] }], + }); + const [call] = rpcCalls('my_review_values'); + assert.ok(call, 'my_review_values must be called'); + assert.equal(call.headers.authorization, `Bearer ${GOOD_TOKEN}`); + assert.deepEqual(call.body, { p_review_ids: [MINE, OTHER] }); + assert.ok( + calls.every((c) => !JSON.stringify(c.headers).includes(SERVICE_KEY)), + 'no request may carry the service-role key' + ); + + // A missing function and other failures are reported, not faked. + rpcResponder = () => ({ + status: 404, + json: { code: 'PGRST202', message: 'Could not find the function' }, + }); + res = await POST( + post('/api/v1/rate/reviews/my-values', { ids: [MINE] }, GOOD_TOKEN) + ); + assert.equal(res.status, 503); + rpcResponder = () => ({ status: 500, json: { code: 'XX000' } }); + res = await POST( + post('/api/v1/rate/reviews/my-values', { ids: [MINE] }, GOOD_TOKEN) + ); + assert.equal(res.status, 502); +} + async function testWatchlist() { const { POST } = await import('../watchlist/route'); type Req = Parameters[0]; @@ -2022,6 +2119,7 @@ async function main() { await testReviews(); await testReviewsV2(); await testReviewsByIdPublishedOnly(); + await testMyReviewValues(); await testWatchlist(); await testCookieSessions(); await testFilterBoundsAndWatchlistPrices(); diff --git a/apps/web/src/app/api/v1/rate/reviews/my-values/route.ts b/apps/web/src/app/api/v1/rate/reviews/my-values/route.ts new file mode 100644 index 00000000..e426fc46 --- /dev/null +++ b/apps/web/src/app/api/v1/rate/reviews/my-values/route.ts @@ -0,0 +1,93 @@ +import { NextResponse } from 'next/server'; +import { + PUBLIC_API_CORS_HEADERS, + payloadTooLargeResponse, + readBoundedJsonBody, +} from '@/server/rate/api-handler'; +import { + authenticateRateRequest, + isMissingRpcError, +} from '@/server/rate/request-auth'; +import { createLogger, serializeError } from '@/lib/logger'; +import { parseMyReviewValue, parseReviewIds } from '@/lib/review-value-display'; + +const log = createLogger('api.reviews.my-values'); + +// The answer belongs to one signed-in writer: never cached, never shared. +const MY_VALUES_HEADERS: Record = { + ...PUBLIC_API_CORS_HEADERS, + 'Access-Control-Allow-Methods': 'POST, OPTIONS', + 'Cache-Control': 'private, no-store', +}; + +export async function OPTIONS() { + return new NextResponse(null, { status: 204, headers: MY_VALUES_HEADERS }); +} + +function fail(status: number, code: string, error: string) { + return NextResponse.json( + { ok: false, error, code }, + { status, headers: MY_VALUES_HEADERS } + ); +} + +/** + * POST { ids: uuid[] }: the value score and gaps of the caller's own reviews + * among `ids` (DOS/Rate#1547, D7). It calls rate.my_review_values with the + * caller's JWT, so the database answers only for reviews they wrote; other + * ids are left out of the answer. + */ +export async function POST(request: Request) { + const auth = await authenticateRateRequest( + request, + 'read your review values', + MY_VALUES_HEADERS + ); + if (!auth.ok) return auth.response; + + const body = await readBoundedJsonBody(request, 8 * 1024); + if (body.tooLarge) return payloadTooLargeResponse(MY_VALUES_HEADERS); + const ids = parseReviewIds( + body.value && typeof body.value === 'object' && !Array.isArray(body.value) + ? (body.value as { ids?: unknown }).ids + : undefined + ); + if (!ids) { + return fail(400, 'INVALID_IDS', 'Provide ids: up to 100 review uuids'); + } + if (ids.length === 0) { + return NextResponse.json( + { ok: true, values: [] }, + { headers: MY_VALUES_HEADERS } + ); + } + + const { data, error } = await auth.rateClient.rpc('my_review_values', { + p_review_ids: ids, + }); + if (error) { + if (isMissingRpcError(error)) { + return fail( + 503, + 'REVIEW_VALUES_UNAVAILABLE', + 'Review values unavailable' + ); + } + log.error('my_review_values failed', { error: serializeError(error) }); + return fail(502, 'READ_FAILED', 'Could not read review values'); + } + + const asked = new Set(ids); + const values = []; + for (const row of Array.isArray(data) ? data : []) { + const id = + typeof row?.review_id === 'string' ? row.review_id.toLowerCase() : ''; + if (!asked.has(id)) continue; + const value = parseMyReviewValue(row.value_score, row.value_breakdown); + if (value) values.push({ id, ...value }); + } + return NextResponse.json( + { ok: true, values }, + { headers: MY_VALUES_HEADERS } + ); +} diff --git a/apps/web/src/app/api/v1/rate/reviews/route.ts b/apps/web/src/app/api/v1/rate/reviews/route.ts index 3eab0e3b..c0437d6d 100644 --- a/apps/web/src/app/api/v1/rate/reviews/route.ts +++ b/apps/web/src/app/api/v1/rate/reviews/route.ts @@ -34,7 +34,8 @@ import { reviewV2FieldFromRpcDetail, submitCommunityReviewV2, } from '@/server/rate/review-v2'; -import { hasRatingCriteria } from '@repo/shared-data/rules'; +import { hasRatingCriteria, storedPoint } from '@repo/shared-data/rules'; +import { evaluateReviewTrustAssessment } from '@/lib/rate-domain'; import type { AuthenticatedRateRequest } from '@/server/rate/request-auth'; const log = createLogger('api.reviews'); @@ -715,18 +716,37 @@ async function submitReviewV2( {} ); const criteria = v2StoredCriteria(record.criteria_ratings); + const reviewWithCriteria: ReviewItem = { + ...review, + criteria_ratings: criteria, + signals: { + ...review.signals, + criteria_count: Object.keys(criteria).length, + }, + }; + // The new card's trust reads the whole v2 text, as the review list will + // (DOS/Rate#1583); the client would otherwise judge the comment alone. + const submission = checked.submission; + reviewWithCriteria.trust_assessment = evaluateReviewTrustAssessment( + reviewWithCriteria, + { + verdict: record.verdict ?? submission.verdict, + // As rate.review_text_stored keeps them, so the list reads the same. + points: [ + ...submission.pros, + ...submission.cons, + ...Object.values(submission.criterionPoints).flatMap((lines) => [ + ...(lines?.pros ?? []), + ...(lines?.cons ?? []), + ]), + ].map(storedPoint), + } + ); return NextResponse.json( { ok: true, mode: 'supabase_rpc', - review: { - ...review, - criteria_ratings: criteria, - signals: { - ...review.signals, - criteria_count: Object.keys(criteria).length, - }, - }, + review: reviewWithCriteria, }, { headers: REVIEWS_CORS_HEADERS } ); diff --git a/apps/web/src/components/reviews/JReviewsSection.tsx b/apps/web/src/components/reviews/JReviewsSection.tsx index dc59f44a..8bdb5864 100644 --- a/apps/web/src/components/reviews/JReviewsSection.tsx +++ b/apps/web/src/components/reviews/JReviewsSection.tsx @@ -6,6 +6,7 @@ import ExportMenu from '@/components/elementary/ExportMenu'; import PhoneCollapsible from '@/components/product/PhoneCollapsible'; import StarRating from '@/components/elementary/StarRating'; import { + BsAward, BsBarChart, BsCheckCircle, BsExclamationTriangle, @@ -14,6 +15,7 @@ import { BsHandThumbsUp, BsInfoCircle, BsMegaphone, + BsPatchCheck, BsRobot, BsStarFill, } from 'react-icons/bs'; @@ -55,6 +57,18 @@ import AnalyzerText, { analyzerText } from './AnalyzerText'; import ReviewFormV2 from './ReviewFormV2'; import { isReviewFormV2Enabled } from '@/lib/review-form-v2'; import { hasRatingCriteria } from '@repo/shared-data/rules'; +import { + hasSessionCookie, + isReviewId, + MY_REVIEW_VALUES_MAX_IDS, + parseMyReviewValue, + compareByQuality, + reviewTierBadgeKey, + reviewValueGapKey, + shownReviewTier, + shownReviewValueGaps, + type MyReviewValue, +} from '@/lib/review-value-display'; import { formatDecimal, formatPercent, @@ -228,6 +242,8 @@ export default function JReviewsSection({ const [authNotice, setAuthNotice] = useState(null); const [activeTab, setActiveTab] = useState('all'); const [sortMode, setSortMode] = useState('weight_desc'); + // The signed-in writer's own value scores, by review id (DOS/Rate#1547). + const [myValues, setMyValues] = useState>({}); const [searchKeyword, setSearchKeyword] = useState(''); const [starFilter, setStarFilter] = useState<'all' | 5 | 4 | 3>('all'); @@ -434,22 +450,8 @@ export default function JReviewsSection({ return [...filtered].sort((a, b) => { if (sortMode === 'weight_desc') { - const wilsonA = computeWilsonLowerBound(a.upvotes, a.downvotes); - const wilsonB = computeWilsonLowerBound(b.upvotes, b.downvotes); - if ( - a.include_in_rating && - b.include_in_rating && - Math.abs(wilsonB - wilsonA) >= 0.005 - ) { - return wilsonB - wilsonA; - } - if (b.helpfulness_weight !== a.helpfulness_weight) { - return b.helpfulness_weight - a.helpfulness_weight; - } - return ( - new Date(b.created_at || 0).getTime() - - new Date(a.created_at || 0).getTime() - ); + // The public quality tier leads (DOS/Rate#1547, D7). + return compareByQuality(a, b, computeWilsonLowerBound); } if (sortMode === 'rating_desc') { return b.rating_star - a.rating_star; @@ -464,6 +466,59 @@ export default function JReviewsSection({ }); }, [reviews, activeTab, sortMode, searchKeyword, starFilter, tAnalyzer]); + // Ids of the reviews on the page, stable across votes, for the lookup below. + const reviewIdsKey = useMemo( + () => + [...new Set(reviews.map((r) => r.id).filter(isReviewId))] + .slice(0, MY_REVIEW_VALUES_MAX_IDS) + .sort() + .join(','), + [reviews] + ); + + // A signed-in writer sees the value score and gaps of their own reviews + // (DOS/Rate#1547, D7). Visitors without a session cookie make no call and + // never load the Supabase client. + useEffect(() => { + let cancelled = false; + void (async () => { + // Without a session, or on any failure, no number stays on screen. + let next: Record = {}; + try { + const token = + reviewIdsKey && hasSessionCookie(document.cookie) + ? await getBrowserAccessToken() + : null; + if (token && !cancelled) { + const res = await fetch('/api/v1/rate/reviews/my-values', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}`, + }, + body: JSON.stringify({ ids: reviewIdsKey.split(',') }), + }); + const payload = res.ok ? await res.json() : null; + for (const row of Array.isArray(payload?.values) + ? payload.values + : []) { + const value = parseMyReviewValue(row?.score, { gaps: row?.gaps }); + if (isReviewId(row?.id) && value) { + next[row.id.toLowerCase()] = value; + } + } + } + } catch { + // The hint is optional; the page works without it. + next = {}; + } + if (!cancelled) setMyValues(next); + })(); + return () => { + cancelled = true; + }; + }, [reviewIdsKey]); + async function handleVote(reviewId: string, clickedVote: VoteChoice) { if (pendingIds[reviewId]) return; @@ -1586,6 +1641,15 @@ export default function JReviewsSection({ const isPending = Boolean(pendingIds[rev.id]); const isZeroWeight = !rev.include_in_rating || rev.helpfulness_weight <= 0; + const tier = shownReviewTier(rev); + const tierKey = reviewTierBadgeKey(tier); + const myValue = myValues[rev.id.toLowerCase()]; + const myGaps = myValue + ? shownReviewValueGaps( + myValue.gaps, + hasRatingCriteria(reviewCategorySlug ?? categorySlug) + ) + : []; return (
@@ -1663,8 +1727,49 @@ export default function JReviewsSection({ ), })} + {tierKey && ( + + {tier === 'excellent' ? ( + + )}
+ {/* The writer's own number and what would raise it; the + public sees only the tier (DOS/Rate#1547, D7). */} + {/* A gated review scores 0 with no gaps: its flag says why. */} + {myValue && (myValue.score > 0 || myGaps.length > 0) && ( +
+ {t('myReviewValue', { score: myValue.score })} + {myGaps.length > 0 && + ` · ${t('myReviewValueGaps', { + gaps: myGaps + .map((gap) => reviewValueGapKey(gap)) + .filter((key) => key !== null) + .map((key) => t(key)) + .join(', '), + })}`} +
+ )} + {rev.variation && (
{t.rich('variationPrefix', { diff --git a/apps/web/src/lib/__tests__/review-value-display.test.ts b/apps/web/src/lib/__tests__/review-value-display.test.ts new file mode 100644 index 00000000..695160e7 --- /dev/null +++ b/apps/web/src/lib/__tests__/review-value-display.test.ts @@ -0,0 +1,147 @@ +import assert from 'node:assert/strict'; +import { + compareByQuality, + hasSessionCookie, + isReviewId, + parseMyReviewValue, + parseReviewIds, + parseReviewValueTier, + reviewTierBadgeKey, + reviewTierRank, + reviewValueGapKey, + shownReviewTier, + shownReviewValueGaps, +} from '../review-value-display'; + +const ID = 'aaaaaaaa-1111-4222-8333-444444444444'; + +// The public tier: only the three stored values. +assert.equal(parseReviewValueTier('excellent'), 'excellent'); +assert.equal(parseReviewValueTier('complete'), 'complete'); +assert.equal(parseReviewValueTier('basic'), 'basic'); +for (const raw of ['Excellent', '', null, undefined, 3, '__proto__']) { + assert.equal(parseReviewValueTier(raw), null); +} + +// Excellent before complete; basic and unscored rank together, last. +assert.ok(reviewTierRank('excellent') > reviewTierRank('complete')); +assert.ok(reviewTierRank('complete') > reviewTierRank('basic')); +assert.equal(reviewTierRank(null), reviewTierRank('basic')); +const sorted = [null, 'excellent', 'basic', 'complete'] + .map((tier) => parseReviewValueTier(tier)) + .sort((a, b) => reviewTierRank(b) - reviewTierRank(a)); +assert.deepEqual(sorted.slice(0, 2), ['excellent', 'complete']); + +// A review the rating leaves out keeps no tier, so it cannot be lifted. +const kept = { value_tier: 'excellent' as const, include_in_rating: true }; +assert.equal(shownReviewTier({ ...kept, helpfulness_weight: 1 }), 'excellent'); +assert.equal(shownReviewTier({ ...kept, helpfulness_weight: 0 }), null); +assert.equal( + shownReviewTier({ ...kept, include_in_rating: false, helpfulness_weight: 1 }), + null +); +assert.equal( + shownReviewTier({ include_in_rating: true, helpfulness_weight: 1 }), + null +); + +// The "most helpful" order: tier first, unless buyers voted it down; then +// votes, weight and the newest. +const wilson = (up: number, down: number) => + up + down === 0 ? 0 : up / (up + down + 2); +const review = ( + id: string, + extra: Partial<{ + value_tier: 'basic' | 'complete' | 'excellent' | null; + include_in_rating: boolean; + helpfulness_weight: number; + upvotes: number; + downvotes: number; + created_at: string; + }> = {} +) => ({ + id, + value_tier: null, + include_in_rating: true, + helpfulness_weight: 1, + upvotes: 0, + downvotes: 0, + created_at: '2026-11-01T00:00:00Z', + ...extra, +}); +const order = (list: ReturnType[]) => + [...list].sort((a, b) => compareByQuality(a, b, wilson)).map((r) => r.id); +assert.deepEqual( + order([ + review('marketplace', { upvotes: 300 }), + review('complete', { value_tier: 'complete' }), + review('excellent', { value_tier: 'excellent' }), + review('voted-down', { value_tier: 'excellent', downvotes: 40 }), + review('flagged', { value_tier: 'excellent', include_in_rating: false }), + ]), + ['excellent', 'complete', 'marketplace', 'voted-down', 'flagged'] +); +assert.deepEqual( + order([ + review('older', { value_tier: 'complete' }), + review('newer', { + value_tier: 'complete', + created_at: '2026-11-02T00:00:00Z', + }), + ]), + ['newer', 'older'] +); + +// The writer sees at most three gaps they can act on. +assert.deepEqual( + shownReviewValueGaps( + ['detail', 'criteria', 'verdict', 'pros_cons', 'photo', 'verified'], + true + ), + ['detail', 'criteria', 'verdict'] +); +assert.deepEqual( + shownReviewValueGaps( + ['criteria', 'criterion_points', 'photo', 'verified'], + false + ), + ['photo'] +); + +// Less text: basic and unscored reviews carry no badge. +assert.equal(reviewTierBadgeKey('excellent'), 'valueTierExcellent'); +assert.equal(reviewTierBadgeKey('complete'), 'valueTierComplete'); +assert.equal(reviewTierBadgeKey('basic'), null); +assert.equal(reviewTierBadgeKey(null), null); + +assert.equal(reviewValueGapKey('pros_cons'), 'valueGapProsCons'); +assert.equal(reviewValueGapKey('toString'), null); +assert.equal(reviewValueGapKey(1), null); + +// Review ids: uuids only, at most 100, deduplicated case-insensitively. +assert.ok(isReviewId(ID)); +assert.ok(!isReviewId('stored')); +assert.deepEqual(parseReviewIds([ID, ID.toUpperCase()]), [ID]); +assert.deepEqual(parseReviewIds([]), []); +assert.equal(parseReviewIds([ID, 'x']), null); +assert.equal(parseReviewIds(ID), null); +assert.equal(parseReviewIds(new Array(101).fill(ID)), null); + +// The writer's value: a 0-100 score, rounded, and the known gaps once each. +assert.deepEqual( + parseMyReviewValue('61.5', { gaps: ['photo', 'x', 'photo', 'verified'] }), + { score: 62, gaps: ['photo', 'verified'] } +); +assert.deepEqual(parseMyReviewValue(0, null), { score: 0, gaps: [] }); +for (const score of [-1, 101, Number.NaN, 'abc', null, undefined]) { + assert.equal(parseMyReviewValue(score, { gaps: [] }), null); +} + +// Only a non-empty Supabase auth cookie counts as a session. +assert.ok(hasSessionCookie('a=1; sb-abc-auth-token=xyz')); +assert.ok(hasSessionCookie('sb-abc-auth-token.0=base64-xyz')); +assert.ok(!hasSessionCookie('sb-abc-auth-token=')); +assert.ok(!hasSessionCookie('x-sb-abc-auth-token=1')); +assert.ok(!hasSessionCookie('')); + +console.log('✅ review value display (tier, ids, writer value) passed'); diff --git a/apps/web/src/lib/__tests__/trust-depth-v2.test.ts b/apps/web/src/lib/__tests__/trust-depth-v2.test.ts new file mode 100644 index 00000000..82a6e22e --- /dev/null +++ b/apps/web/src/lib/__tests__/trust-depth-v2.test.ts @@ -0,0 +1,75 @@ +import assert from 'node:assert/strict'; +import { evaluateReviewTrustAssessment } from '../rate-domain'; + +// A form v2 review: its verdict and its writer's pros and cons count with +// the comment for axis C; AI-extracted lines do not (DOS/Rate#1583). +const v2Review = { + rating_star: 5, + comment: '', + reviewer_type: 'user' as const, + upvotes: 0, + downvotes: 0, + report_count: 0, + flag_category: 'authentic' as const, + verdict: 'authentic' as const, + include_in_rating: true, + signals: null, +}; +const v2Text = { + verdict: 'Máy mượt, đáng tiền trong tầm giá.', + points: [ + { + kind: 'pro', + criterion_key: null, + body: 'Pin dùng cả ngày', + method: 'writer', + }, + { + kind: 'con', + criterion_key: 'battery', + body: 'Sạc hơi chậm', + method: 'writer', + }, + ], +}; +const v2Bare = evaluateReviewTrustAssessment(v2Review); +const v2Eval = evaluateReviewTrustAssessment(v2Review, v2Text); +assert.equal(v2Bare.depth, 'stars_only'); +assert.equal(v2Eval.depth, 'specific_short'); +assert.ok(v2Eval.breakdown.weight > v2Bare.breakdown.weight); +assert.equal( + evaluateReviewTrustAssessment(v2Review, { + points: v2Text.points.map((p) => ({ ...p, method: 'ai' })), + }).depth, + 'stars_only' +); +// The writer's lines count on their own: without a verdict, two lines +// that name the product reach specific_short. +assert.equal( + evaluateReviewTrustAssessment(v2Review, { + points: [ + { kind: 'pro', body: 'Pin dùng được cả ngày', method: 'writer' }, + { kind: 'con', body: 'Sạc hơi chậm nhưng ổn', method: 'writer' }, + ], + }).depth, + 'specific_short' +); +// v2 stores the verdict as the comment when there is no free text: said +// twice, it counts once. Doubled, this 53-character verdict would reach +// the 90 of `detailed`. +const longVerdict = 'Máy mượt, pin dùng hết ngày, đáng tiền trong tầm giá.'; +const once = evaluateReviewTrustAssessment({ + ...v2Review, + comment: longVerdict, +}); +const twice = evaluateReviewTrustAssessment( + { ...v2Review, comment: longVerdict }, + { verdict: longVerdict, points: null } +); +assert.equal(once.depth, 'specific_short'); +assert.equal(twice.depth, 'specific_short'); +assert.equal(twice.breakdown.weight, once.breakdown.weight); + +console.log( + 'trust-depth-v2: the verdict and pros and cons of a v2 review count for axis C' +); diff --git a/apps/web/src/lib/rate-api.ts b/apps/web/src/lib/rate-api.ts index 113cb708..5ad5e157 100644 --- a/apps/web/src/lib/rate-api.ts +++ b/apps/web/src/lib/rate-api.ts @@ -4,6 +4,7 @@ import { createLogger } from '@/lib/logger'; import { timeStep } from '@/lib/server-timing'; import { rankModelCandidates } from '@/lib/model-match'; import { isPendingReview } from '@/lib/review-display'; +import { parseReviewValueTier } from '@/lib/review-value-display'; import { listingReviewSummary } from '@/lib/analyzer-copy'; import { countBrandsByItem } from '@/lib/item-directory'; import { masterSiblingKeyPrefix } from '@/lib/product-variant-links'; @@ -1688,6 +1689,20 @@ interface RawReviewFlagRow { signals?: Record | null; } +/** + * A review row with its analyzer flag and the writer's pros and cons + * (form v2), which axis C reads with the comment (DOS/Rate#1583). + */ +const REVIEW_SELECT = + '*, review_flags(*), review_points(kind, criterion_key, body, method)'; + +interface RawReviewPointRow { + kind?: string | null; + criterion_key?: string | null; + body?: string | null; + method?: string | null; +} + interface RawReviewRow { id: string; author_name?: string | null; @@ -1702,6 +1717,11 @@ interface RawReviewRow { helpfulness_weight?: number | string | null; criteria_ratings?: Record | null; review_flags?: RawReviewFlagRow | RawReviewFlagRow[] | null; + /** The form v2 verdict line; the analyzer's verdict is in review_flags. */ + verdict?: string | null; + review_points?: RawReviewPointRow[] | null; + /** The public quality tier (DOS/Rate#1547). */ + value_tier?: string | null; } /** @@ -1820,7 +1840,7 @@ export async function getMasterProductReviews( (chunk) => supabase .from('reviews') - .select('*, review_flags(*)') + .select(REVIEW_SELECT) .in('product_id', chunk) .order('created_at', { ascending: false }) .limit(50) @@ -1890,8 +1910,12 @@ export async function getMasterProductReviews( flag_category: flagCategory, reason: vf?.reason || null, signals: vf?.signals || null, + value_tier: parseReviewValueTier(r.value_tier), }; - item.trust_assessment = evaluateReviewTrustAssessment(item); + item.trust_assessment = evaluateReviewTrustAssessment(item, { + verdict: r.verdict, + points: r.review_points, + }); return item; }); } @@ -3324,6 +3348,7 @@ export async function getMerchantsList( flag_category: flagCategory, reason: vf?.reason || null, signals: vf?.signals || null, + value_tier: parseReviewValueTier(r.value_tier), }; const existing = mappedReviewsByMerchant.get(mId) ?? []; @@ -3717,7 +3742,7 @@ export async function getMerchantProfileBase( (chunk) => supabase .from('reviews') - .select('*, review_flags(*)') + .select(REVIEW_SELECT) .in('product_id', chunk) .order('created_at', { ascending: false }) .limit(60) @@ -3887,8 +3912,12 @@ export async function getMerchantProfileBase( flag_category: flagCategory, reason: vf?.reason || null, signals: vf?.signals || null, + value_tier: parseReviewValueTier(r.value_tier), }; - item.trust_assessment = evaluateReviewTrustAssessment(item); + item.trust_assessment = evaluateReviewTrustAssessment(item, { + verdict: r.verdict, + points: r.review_points, + }); return item; }); @@ -4134,7 +4163,7 @@ export async function getRatePlatformStats(): Promise { supabase.from('reviews').select('id', { count: 'exact', head: true }), supabase .from('reviews') - .select('*, review_flags(*)') + .select(REVIEW_SELECT) .order('created_at', { ascending: false }) .limit(60), // Platform-wide review verdict counts (DOS/Rate#455), with the same rules @@ -4327,7 +4356,10 @@ export async function getRatePlatformStats(): Promise { masterSlug: pInfo?.masterSlug || null, masterRouteHref: pInfo?.masterRouteHref || null, }; - item.trust_assessment = evaluateReviewTrustAssessment(item); + item.trust_assessment = evaluateReviewTrustAssessment(item, { + verdict: r.verdict, + points: r.review_points, + }); return item; }); diff --git a/apps/web/src/lib/rate-domain-core.ts b/apps/web/src/lib/rate-domain-core.ts index c06108d7..03d8160b 100644 --- a/apps/web/src/lib/rate-domain-core.ts +++ b/apps/web/src/lib/rate-domain-core.ts @@ -5,6 +5,7 @@ * them (DOS/Rate#756). */ import type { MasterProduct } from '@repo/shared-data'; +import type { ReviewValueTier } from '@repo/shared-data/rules'; import { REVIEW_PLATFORM_LABEL_RE, REVIEW_PRODUCT_ASPECT_RE, @@ -148,6 +149,8 @@ export interface ReviewItem { reason: string | null; signals: Record | null; trust_assessment?: ReviewTrustAssessment; + /** The public quality tier (DOS/Rate#1547, D7); null when unscored. */ + value_tier?: ReviewValueTier | null; } export function toSafeNonNegativeVoteCount( @@ -204,7 +207,7 @@ export function computeWilsonLowerBound( const WEB_SHIPPING_ASPECT_RE = /(?:giao\s*hàng|ship(?:per)?|vận\s*chuyển|đóng\s*gói|bọc\s*hàng|hộp\s*móp|giao\s*nhanh|giao\s*chậm|tư\s*vấn|chăm\s*sóc\s*khách)/i; -// Any mention of the product or of using it, based on PRODUCT_ASPECT_RE in +// Any mention of the product or of using it, based on the product words in // apps/importer/src/shopee/ai-review-analyzer.ts (isShippingOnlyComment) plus // Shopee's structured product fields. Matched as whole words so "hư" in "như" // or "pin" in "shopping" do not count. Words that also describe delivery @@ -245,7 +248,13 @@ export function evaluateReviewTrustAssessment( | 'verdict' | 'include_in_rating' | 'signals' - > + >, + /** + * The rest of a form v2 review's text, which axis C reads with the + * comment: its verdict line and its writer's pros and cons + * (`rate.review_points`, DOS/Rate#1583). + */ + reviewText?: { verdict?: unknown; points?: readonly unknown[] | null } ): ReviewTrustAssessment { const review = ( rawReview && typeof rawReview === 'object' && !Array.isArray(rawReview) @@ -310,6 +319,8 @@ export function evaluateReviewTrustAssessment( // 3. Axis C: Depth, shared with the review value score (DOS/Rate#1581). const depth: ReviewDepthAxis = reviewDepth({ comment, + verdict: reviewText?.verdict, + points: reviewText?.points ?? undefined, reviewerType: review.reviewer_type, flagCategory: review.flag_category, }); diff --git a/apps/web/src/lib/review-value-display.ts b/apps/web/src/lib/review-value-display.ts new file mode 100644 index 00000000..2879a332 --- /dev/null +++ b/apps/web/src/lib/review-value-display.ts @@ -0,0 +1,192 @@ +import type { ReviewValueGap, ReviewValueTier } from '@repo/shared-data/rules'; + +/** + * Review value on the web (DOS/Rate#1547, D7). The public sees a review's + * tier (`rate.reviews.value_tier`); only the writer sees the number and what + * would raise it, read through `rate.my_review_values` as that writer. + */ + +const TIERS: readonly ReviewValueTier[] = ['basic', 'complete', 'excellent']; + +/** The tier of a stored review, null when it is unscored, gated or unknown. */ +export function parseReviewValueTier(raw: unknown): ReviewValueTier | null { + return typeof raw === 'string' && (TIERS as readonly string[]).includes(raw) + ? (raw as ReviewValueTier) + : null; +} + +/** Sort rank of a tier: an unscored review ranks with a basic one. */ +export function reviewTierRank(tier: ReviewValueTier | null | undefined) { + return tier === 'excellent' ? 2 : tier === 'complete' ? 1 : 0; +} + +/** + * The tier a review shows and sorts by. A review the rating leaves out (a + * flag or zero weight since it was scored) shows none, so a stale tier never + * lifts it. + */ +export function shownReviewTier(review: { + value_tier?: ReviewValueTier | null; + include_in_rating: boolean; + helpfulness_weight: number; +}): ReviewValueTier | null { + return review.include_in_rating && review.helpfulness_weight > 0 + ? (review.value_tier ?? null) + : null; +} + +interface QualitySortedReview { + value_tier?: ReviewValueTier | null; + include_in_rating: boolean; + helpfulness_weight: number; + upvotes: number; + downvotes: number; + created_at: string | null; +} + +/** Buyers voted it down: more down than up votes, at least 3 in all. */ +function votedDown(r: { upvotes: number; downvotes: number }): boolean { + return r.downvotes > r.upvotes && r.upvotes + r.downvotes >= 3; +} + +/** + * The "most helpful" order: the shown tier first (design 6.3), unless buyers + * voted the review down, so votes still outrank a complete form; then the + * helpful-vote order and the newest. `wilson` is the Wilson lower bound of a + * review's votes. + */ +export function compareByQuality( + a: T, + b: T, + wilson: (upvotes: number, downvotes: number) => number +): number { + const rank = (r: T) => + votedDown(r) ? 0 : reviewTierRank(shownReviewTier(r)); + const tierGap = rank(b) - rank(a); + if (tierGap !== 0) return tierGap; + const wilsonA = wilson(a.upvotes, a.downvotes); + const wilsonB = wilson(b.upvotes, b.downvotes); + if ( + a.include_in_rating && + b.include_in_rating && + Math.abs(wilsonB - wilsonA) >= 0.005 + ) { + return wilsonB - wilsonA; + } + if (b.helpfulness_weight !== a.helpfulness_weight) { + return b.helpfulness_weight - a.helpfulness_weight; + } + return ( + new Date(b.created_at || 0).getTime() - + new Date(a.created_at || 0).getTime() + ); +} + +/** + * The gaps the writer is shown: at most three they can act on. A verified + * purchase has no path from the form yet, and criteria gaps need a category + * with a criteria list. + */ +export function shownReviewValueGaps( + gaps: readonly ReviewValueGap[], + hasCriteria: boolean +): ReviewValueGap[] { + return gaps + .filter( + (gap) => + gap !== 'verified' && + (hasCriteria || (gap !== 'criteria' && gap !== 'criterion_points')) + ) + .slice(0, 3); +} + +/** The badge a tier shows: basic and unscored show none (less text). */ +export function reviewTierBadgeKey( + tier: ReviewValueTier | null | undefined +): 'valueTierExcellent' | 'valueTierComplete' | null { + return tier === 'excellent' + ? 'valueTierExcellent' + : tier === 'complete' + ? 'valueTierComplete' + : null; +} + +const GAP_KEYS = { + detail: 'valueGapDetail', + criteria: 'valueGapCriteria', + verdict: 'valueGapVerdict', + pros_cons: 'valueGapProsCons', + criterion_points: 'valueGapCriterionPoints', + photo: 'valueGapPhoto', + verified: 'valueGapVerified', +} as const satisfies Record; + +export type ReviewValueGapKey = (typeof GAP_KEYS)[ReviewValueGap]; + +/** The locale key naming a gap, null for a name this build does not know. */ +export function reviewValueGapKey(gap: unknown): ReviewValueGapKey | null { + return typeof gap === 'string' && Object.hasOwn(GAP_KEYS, gap) + ? GAP_KEYS[gap as ReviewValueGap] + : null; +} + +/** True for a review id the value lookup can carry. */ +export function isReviewId(id: unknown): id is string { + return typeof id === 'string' && UUID_RE.test(id); +} + +/** What the writer of a review sees about its value. */ +export interface MyReviewValue { + score: number; + gaps: ReviewValueGap[]; +} + +/** At most this many ids per call, as `rate.my_review_values` allows. */ +export const MY_REVIEW_VALUES_MAX_IDS = 100; + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** The distinct review ids of a request, null when any is not a uuid. */ +export function parseReviewIds(raw: unknown): string[] | null { + if (!Array.isArray(raw) || raw.length > MY_REVIEW_VALUES_MAX_IDS) { + return null; + } + const ids = new Set(); + for (const id of raw) { + if (!isReviewId(id)) return null; + ids.add(id.toLowerCase()); + } + return [...ids]; +} + +/** + * One row of `rate.my_review_values` (or of the route's answer) as the + * writer's value; null for a row that is not a scored review. + */ +export function parseMyReviewValue( + score: unknown, + breakdown: unknown +): MyReviewValue | null { + const n = typeof score === 'string' ? Number(score) : score; + if (typeof n !== 'number' || !Number.isFinite(n) || n < 0 || n > 100) { + return null; + } + const rawGaps = + breakdown && typeof breakdown === 'object' && !Array.isArray(breakdown) + ? (breakdown as { gaps?: unknown }).gaps + : undefined; + const gaps = Array.isArray(rawGaps) + ? (rawGaps.filter((g) => reviewValueGapKey(g) !== null) as ReviewValueGap[]) + : []; + return { score: Math.round(n), gaps: [...new Set(gaps)] }; +} + +/** True when the browser holds a Supabase session cookie. */ +export function hasSessionCookie(cookie: string): boolean { + return typeof cookie === 'string' + ? /(?:^|;\s*)sb-[a-z0-9_-]{1,64}-auth-token(?:\.\d{1,3})?=[^;\s]/i.test( + cookie + ) + : false; +} diff --git a/apps/web/src/server/rate/openapi.ts b/apps/web/src/server/rate/openapi.ts index 6d7235c2..a7971fcc 100644 --- a/apps/web/src/server/rate/openapi.ts +++ b/apps/web/src/server/rate/openapi.ts @@ -1693,6 +1693,43 @@ function buildBaseRateOpenApiSpec( }, }, }, + '/api/v1/rate/reviews/my-values': { + post: { + operationId: 'getMyReviewValues', + summary: "Read the value score of the caller's own reviews", + description: + "Returns the 0-100 value score and the missing parts (`gaps`) of the reviews among `ids` that the signed-in caller wrote, through `rate.my_review_values` with the caller's JWT. Other ids are left out. The public sees only each review's `value_tier`. Never cached.", + tags: ['Community Reviews & Voting'], + security: [{ bearerAuth: [] }], + requestBody: { + required: true, + content: { + 'application/json': { + schema: { + type: 'object', + required: ['ids'], + properties: { + ids: { + type: 'array', + maxItems: 100, + items: { type: 'string', format: 'uuid' }, + }, + }, + }, + }, + }, + }, + responses: { + '200': { + description: + "`values`: `{ id, score, gaps }` for each of the caller's own reviews that has a score.", + }, + '400': errorResponse('`ids` is not a list of up to 100 uuids.'), + '401': errorResponse('Missing or invalid bearer token.'), + '503': errorResponse('Review values are not available here.'), + }, + }, + }, '/api/v1/rate/stats': { get: { operationId: 'getPlatformStats', @@ -2359,6 +2396,12 @@ function buildBaseRateOpenApiSpec( ], }, reason: { type: ['string', 'null'] }, + value_tier: { + type: ['string', 'null'], + enum: ['basic', 'complete', 'excellent', null], + description: + 'Public quality tier of the review; null when it is not scored yet or is gated. The 0-100 value score is private to the writer.', + }, }, }, }, diff --git a/packages/shared-data/src/__tests__/shared-data.test.ts b/packages/shared-data/src/__tests__/shared-data.test.ts index c4e34935..8ddfadf0 100644 --- a/packages/shared-data/src/__tests__/shared-data.test.ts +++ b/packages/shared-data/src/__tests__/shared-data.test.ts @@ -2084,6 +2084,59 @@ function testReviewDepth(): void { rules.reviewDepthText(v2), 'Máy mượt, đáng tiền trong tầm giá.\nPin dùng cả ngày\nSạc hơi chậm' ); + // A measured battery report is technical depth (DOS/Rate#1583). + for (const line of [ + 'Pin dùng được 7 tiếng liên tục, lướt web và xem video', + 'SOT 8 giờ với 5G bật cả ngày, pin ổn hơn máy cũ', + 'Pin trâu sot 9h, chơi game nhẹ buổi tối vẫn còn', + 'Dùng 6h on screen vẫn còn 20%, rất hài lòng', + 'Sạc từ 20% lên 100% mất 45 phút, sạc khá nhanh', + 'Sạc 0-100% 50p, máy hơi ấm khi sạc nhanh', + 'Test thực tế pin trụ 9 tiếng với độ sáng tự động', + 'Màn hình bật 7 tiếng còn 30%, quá ổn cho nhu cầu', + 'Chơi game 5 tiếng tụt 40% pin, máy không nóng lắm', + 'Dùng liên tục 6-7 tiếng mới phải sạc lại, pin ổn', + 'Screen-on time 9 hours on Wi-Fi, very good battery', + 'Onscreen ~7h với 4G, pin tốt trong tầm giá', + ]) { + assert.equal(depth({ comment: line }), 'expert_depth', line); + } + // Hours that are not a battery report, and accessories, are not one. + for (const line of [ + 'Giao hàng 2 tiếng là tới, đóng gói cẩn thận', + 'Đợi 3 tiếng mới được tư vấn, nhân viên chậm', + 'Pin tốt, giao hàng sau 3 tiếng', + 'pin ổn, chờ 2 tiếng mới có hàng', + 'Mua kèm pin dự phòng 10000, giao 2h chiều', + 'Pin con thỏ mua 2 hộp giao 3h', + 'Pin chưa dùng thử, mới mở hộp 1h', + 'Đèn pin sáng 5h liền, cầm chắc tay', + 'Shop test thực tế trước khi giao, rất nhiệt tình', + 'Đo thực tế vừa vặn, áo mặc vừa người', + 'sot 1 sao vì giao chậm quá', + ]) { + assert.notEqual(depth({ comment: line }), 'expert_depth', line); + } + // A measured line counts in a v2 review's pros too. + assert.equal( + depth({ + verdict: 'Máy ổn trong tầm giá, hợp nhu cầu', + points: ['SOT khoảng 7 tiếng mỗi ngày'], + }), + 'expert_depth' + ); + // A bare measurement shorter than specific_short is not expert depth. + assert.notEqual(depth({ comment: 'Pin 7 tiếng' }), 'expert_depth'); + // Long digit runs are scanned once (no quadratic backtracking). + const started = Date.now(); + depth({ comment: `sạc 1% - 1% ${'1'.repeat(100_000)}` }); + depth({ comment: '1'.repeat(100_000) }); + assert.ok(Date.now() - started < 2_000, 'digit runs stay linear'); + // Padding still adds nothing: "pin ok" repeated is no measurement. + assert.equal( + depth({ comment: Array(40).fill('pin ok').join(' ') }), + 'low_effort' + ); // A number ending one part and a unit starting the next are no measurement. assert.equal(depth({ comment: 'Đặt 2', verdict: 'gọn nhẹ' }), 'low_effort'); // Lines extracted by AI from a marketplace comment add nothing. diff --git a/packages/shared-data/src/database.types.ts b/packages/shared-data/src/database.types.ts index 32c0907d..a45ea342 100644 --- a/packages/shared-data/src/database.types.ts +++ b/packages/shared-data/src/database.types.ts @@ -943,6 +943,34 @@ export type Database = { }, ]; }; + review_values: { + Row: { + review_id: string; + updated_at: string; + value_breakdown: Json; + value_score: number; + }; + Insert: { + review_id: string; + updated_at?: string; + value_breakdown: Json; + value_score: number; + }; + Update: { + review_id?: string; + updated_at?: string; + value_breakdown?: Json; + value_score?: number; + }; + Relationships: [ + { + foreignKeyName: 'review_values_review_id_fkey'; + columns: ['review_id']; + referencedRelation: 'reviews'; + referencedColumns: ['id']; + }, + ]; + }; review_votes: { Row: { created_at: string; @@ -1006,6 +1034,7 @@ export type Database = { upvotes: number | null; value_breakdown: Json | null; value_score: number | null; + value_tier: string | null; variation: string | null; verdict: string | null; }; @@ -1034,6 +1063,7 @@ export type Database = { upvotes?: number | null; value_breakdown?: Json | null; value_score?: number | null; + value_tier?: string | null; variation?: string | null; verdict?: string | null; }; @@ -1062,6 +1092,7 @@ export type Database = { upvotes?: number | null; value_breakdown?: Json | null; value_score?: number | null; + value_tier?: string | null; variation?: string | null; verdict?: string | null; }; @@ -1177,6 +1208,42 @@ export type Database = { }; Returns: number; }; + crawl_job_failures: { + Args: { p_limit?: number; p_since?: string }; + Returns: { + error_kind: string; + jobs: number; + latest_updated_at: string; + platform: string; + status: string; + type: string; + }[]; + }; + crawl_job_repeat_failures: { + Args: { p_limit?: number; p_min_failures?: number; p_since?: string }; + Returns: { + failures: number; + first_failed_at: string; + last_failed_at: string; + latest_error_kind: string; + platform: string; + platform_item_id: string; + }[]; + }; + crawl_queue_summary: { + Args: { p_platforms?: string[] }; + Returns: { + jobs: number; + newest_updated_at: string; + oldest_created_at: string; + pending_in_backoff: number; + platform: string; + priority: number; + running_lease_expired: number; + status: string; + type: string; + }[]; + }; current_user_can_vote_reviews: { Args: never; Returns: boolean }; dequeue_crawl_job: { Args: { p_lease?: string; p_platform: string; p_worker_id: string }; @@ -1325,6 +1392,14 @@ export type Database = { total_reviews: number; }[]; }; + my_review_values: { + Args: { p_review_ids: string[] }; + Returns: { + review_id: string; + value_breakdown: Json; + value_score: number; + }[]; + }; purge_completed_crawl_jobs: { Args: { p_retention_days?: number }; Returns: number; @@ -1371,6 +1446,10 @@ export type Database = { input_hash: string; }[]; }; + review_completeness_factor: { + Args: { p_breakdown: Json }; + Returns: number; + }; review_input_hash: { Args: { p_author: string; p_comment: string; p_rating: number }; Returns: string; @@ -1487,6 +1566,7 @@ export type Database = { Args: { p_params?: Json; p_source: string }; Returns: string; }; + store_review_values: { Args: { p_values: Json }; Returns: number }; submit_community_review: { Args: { p_author_name: string; diff --git a/packages/shared-data/src/rules/review-depth.ts b/packages/shared-data/src/rules/review-depth.ts index 86bb5202..d43e1bea 100644 --- a/packages/shared-data/src/rules/review-depth.ts +++ b/packages/shared-data/src/rules/review-depth.ts @@ -33,6 +33,48 @@ export const REVIEW_PRODUCT_ASPECT_RE = export const REVIEW_TECH_MEASURE_RE = /(?:\b\d+\s*(?:mah|wh|w|hz|nit|nits|fps|gb|tb|mp|mm|gram|g|db|ms|°c|độ\s*c)\b|geekbench|antutu|3dmark|cinebench|pcmag|gsmarena|phonearena)/i; +// Pieces of REVIEW_BATTERY_MEASURE_RE. A number never starts inside another +// number, so a long run of digits is scanned once. Every gap stays inside one +// clause: it stops at punctuation and at a delivery or waiting word. +const NOT_WORD_BEFORE = String.raw`(? REVIEW_TECH_MEASURE_RE.test(part)) || + (length >= REVIEW_DEPTH_LENGTH.specificShort && + parts.some((part) => REVIEW_BATTERY_MEASURE_RE.test(part))) || (length >= REVIEW_DEPTH_LENGTH.expert && hasProduct) ) { return 'expert_depth'; diff --git a/scripts/test-criteria-ratings-proposal.mjs b/scripts/test-criteria-ratings-proposal.mjs index 418ccf3c..121ac46a 100644 --- a/scripts/test-criteria-ratings-proposal.mjs +++ b/scripts/test-criteria-ratings-proposal.mjs @@ -54,6 +54,9 @@ const review = (n) => `d0000000-0000-4000-8000-00000000000${n}`; try { const { sql, asRole } = startRateDb({ name: container }); + // The reference stack has P6 (20261102233000), which sits on P2: roll it back first, as + // production would before a P2 rollback. + sql(read('2026-11-02-dos-me-review-value-score-rollback.sql')); const checkAll = (when) => { for (const { id, query } of checks) assert.equal(sql(query), '0', `${id} ${when}`); diff --git a/scripts/test-review-value-proposal.mjs b/scripts/test-review-value-proposal.mjs index 51c66369..cf52967a 100644 --- a/scripts/test-review-value-proposal.mjs +++ b/scripts/test-review-value-proposal.mjs @@ -139,6 +139,12 @@ const expectedScore = (lifted) => { try { const { sql, asRole } = startRateDb({ name: container }); + // The reference stack has P6 (20261102233000): roll it back, with the CHECK the rollback + // keeps, so the proposal replays on the stack before it. + sql(rollback); + sql( + `ALTER TABLE rate.reviews DROP CONSTRAINT IF EXISTS reviews_value_private` + ); const checkAll = (when) => { for (const { id, query } of checks) assert.equal(sql(query), '0', `${id} ${when}`); diff --git a/supabase/migrations/20261030140000_rate_worker_crawl_queue_reads.sql b/supabase/migrations/20261030140000_rate_worker_crawl_queue_reads.sql new file mode 100644 index 00000000..67e23e71 --- /dev/null +++ b/supabase/migrations/20261030140000_rate_worker_crawl_queue_reads.sql @@ -0,0 +1,279 @@ +-- ============================================================================= +-- Migration: 20261030140000_rate_worker_crawl_queue_reads.sql +-- Purpose (DOS/Rate-Services#739, DOS/DOS.Me#1460): read-only aggregates over +-- the crawl queue for the Rate-Services worker role. +-- +-- rate_worker has no grant on rate.crawl_jobs (20261018183000) and must not get +-- one: claim_token fences ack_crawl_job / fail_crawl_job, and payload, progress +-- and target_url hold page data. Its audits only need counts, ages and error +-- groups, so they read the queue through three SECURITY DEFINER functions that +-- return nothing else: +-- +-- rate.crawl_queue_summary(p_platforms) - jobs per (platform, type, +-- status, priority) with the oldest created_at, newest updated_at, pending +-- jobs in a retry backoff and running jobs whose lease ran out; at most 500 +-- rows. p_platforms NULL means every platform. +-- rate.crawl_job_failures(p_since, p_limit) - failed and skipped jobs per +-- (status, platform, type, error kind) updated within p_since; at most 500 +-- rows (p_limit, default 200). +-- rate.crawl_job_repeat_failures(p_min_failures, p_since, p_limit) - listings +-- (platform, public platform_item_id) with at least p_min_failures failed or +-- skipped jobs whose error is a connection error, with the first and last +-- failure and the latest error kind; at most 100 rows (p_limit, default 50). +-- +-- Windows are clamped to 1 hour .. 14 days: purge_completed_crawl_jobs keeps +-- terminal rows 14 days by default (20261018120105), so a longer window would +-- read a partial history. Raise the clamp together with the retention. +-- +-- error_kind is the first 1000 characters of last_error normalized in SQL +-- (each distinct text once) so raw error text never leaves the database: +-- URLs, double- and single-quoted strings (an unclosed quote to the end; +-- b'', r'' and u'' literals included; an apostrophe inside a word such as +-- can't is not a quote), cookie headers, +-- user:password@ credentials, email addresses, UUIDs, JWTs and other 32+ +-- character tokens, IPv4 hosts, numbers of 5+ digits, bearer / basic +-- credentials and key=value secrets are replaced, whitespace and control +-- characters collapse, and the result is cut at 200 characters. Callers still +-- normalize it again (Rate-Services crawlJobErrorKind + redactSensitiveString) +-- and re-sum groups that become equal. platform_item_id is cut at 200 too. +-- +-- Statement timeout: a function-level SET statement_timeout does not bound the +-- statement that calls the function (the timer starts with the outer +-- statement), so none is set here. A PostgREST call runs under the request +-- role's statement_timeout, or authenticator's when that role sets none. +-- +-- EXECUTE goes to service_role and rate_worker only. Nothing is granted on any +-- table, and no data changes. +-- Idempotent: an existing function with another signature or result type is +-- dropped first (CREATE OR REPLACE cannot change a result type), then CREATE OR +-- REPLACE, REVOKE and GRANT. +-- ============================================================================= + +BEGIN; + +-- Fail fast instead of queueing behind long transactions on the shared database. +SET LOCAL lock_timeout = '5s'; + +DO $$ +DECLARE + r RECORD; +BEGIN + FOR r IN + SELECT p.oid::regprocedure AS sig + FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace + LEFT JOIN (VALUES + ('crawl_queue_summary', + 'p_platforms text[]', + 'TABLE(platform text, type text, status text, priority integer, jobs bigint, oldest_created_at timestamp with time zone, newest_updated_at timestamp with time zone, pending_in_backoff bigint, running_lease_expired bigint)'), + ('crawl_job_failures', + 'p_since interval, p_limit integer', + 'TABLE(status text, platform text, type text, error_kind text, jobs bigint, latest_updated_at timestamp with time zone)'), + ('crawl_job_repeat_failures', + 'p_min_failures integer, p_since interval, p_limit integer', + 'TABLE(platform text, platform_item_id text, failures bigint, first_failed_at timestamp with time zone, last_failed_at timestamp with time zone, latest_error_kind text)') + ) AS want(proname, args, result) + ON want.proname = p.proname + AND want.args = pg_get_function_identity_arguments(p.oid) + AND want.result = pg_get_function_result(p.oid) + WHERE n.nspname = 'rate' + AND p.proname IN ('crawl_queue_summary', 'crawl_job_failures', 'crawl_job_repeat_failures') + AND want.proname IS NULL + LOOP + EXECUTE format('DROP FUNCTION %s', r.sig); + END LOOP; +END $$; + +-- A. Queue summary: counts and ages only. +CREATE OR REPLACE FUNCTION rate.crawl_queue_summary(p_platforms text[] DEFAULT NULL) +RETURNS TABLE ( + platform text, + type text, + status text, + priority integer, + jobs bigint, + oldest_created_at timestamptz, + newest_updated_at timestamptz, + pending_in_backoff bigint, + running_lease_expired bigint +) +LANGUAGE sql +STABLE +SECURITY DEFINER +SET search_path = '' +AS $$ + SELECT j.platform, + j.type, + j.status, + j.priority, + pg_catalog.count(*)::bigint, + pg_catalog.min(j.created_at), + pg_catalog.max(j.updated_at), + (pg_catalog.count(*) FILTER (WHERE j.status = 'pending' AND j.backoff_until > pg_catalog.now()))::bigint, + (pg_catalog.count(*) FILTER (WHERE j.status = 'running' AND j.lease_expires < pg_catalog.now()))::bigint + FROM rate.crawl_jobs j + WHERE p_platforms IS NULL OR j.platform = ANY (p_platforms) + GROUP BY j.platform, j.type, j.status, j.priority + ORDER BY j.platform, j.type, j.status, j.priority + LIMIT 500; +$$; + +-- B. Failed and skipped jobs by (status, platform, type, error kind). +-- The error_kind expression is the same in C; keep them in step. +CREATE OR REPLACE FUNCTION rate.crawl_job_failures( + p_since interval DEFAULT interval '3 days', + p_limit integer DEFAULT 200 +) +RETURNS TABLE ( + status text, + platform text, + type text, + error_kind text, + jobs bigint, + latest_updated_at timestamptz +) +LANGUAGE sql +STABLE +SECURITY DEFINER +SET search_path = '' +AS $$ + SELECT f.status, f.platform, f.type, f.error_kind, + pg_catalog.sum(f.n)::bigint AS jobs, + pg_catalog.max(f.latest) AS latest_updated_at + FROM ( + -- Normalize each distinct error once, not once per job. + SELECT r.status, r.platform, r.type, r.n, r.latest, + pg_catalog.left( + pg_catalog.btrim( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + r.raw, + '[a-z][a-z0-9+.-]*://\S+', '', 'gi'), + '"[^"]*("|$)', '"\1', 'g'), + '(?\2', 'gi'), + '\m((set-)?cookie)\s*[=:][^\r\n]*', '\1=', 'gi'), + '[^\s/@:"<>]+:[^\s/@"<>]*@', '@', 'g'), + '[a-z0-9._%+-]+@[a-z0-9-]+(\.[a-z0-9-]+)+', '', 'gi'), + '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', '', 'gi'), + 'eyJ[a-z0-9_-]{8,}(\.[a-z0-9_-]+){0,2}', '', 'gi'), + '[a-z0-9_-]{32,}', '', 'gi'), + '\m[0-9]{1,3}(\.[0-9]{1,3}){3}(:[0-9]+)?', '', 'g'), + '[0-9]{5,}', '#', 'g'), + '\m(bearer|basic)\s+[^\s,;]+', '\1 ', 'gi'), + '\m([a-z_-]*(token|key|secret|password|passwd|signature|authorization|session|credential)[a-z_-]*)\s*[=:]\s*[^\s,;&]+', '\1=', 'gi'), + '[[:space:][:cntrl:]]+', ' ', 'g')), + 200) AS error_kind + FROM ( + SELECT j.status, j.platform, j.type, + pg_catalog.left(COALESCE(j.last_error, ''), 1000) AS raw, + pg_catalog.count(*) AS n, + pg_catalog.max(j.updated_at) AS latest + FROM rate.crawl_jobs j + WHERE j.status IN ('failed', 'skipped') + AND j.updated_at >= pg_catalog.now() - LEAST(GREATEST(COALESCE(p_since, interval '3 days'), interval '1 hour'), interval '14 days') + GROUP BY 1, 2, 3, 4 + ) r + ) f + GROUP BY f.status, f.platform, f.type, f.error_kind + ORDER BY jobs DESC, latest_updated_at DESC, f.status, f.platform, f.type, f.error_kind + LIMIT LEAST(GREATEST(COALESCE(p_limit, 200), 1), 500); +$$; + +-- C. Listings with repeated connection failures: the public marketplace item +-- id, counts, timestamps and the latest error kind (normalized as in B). +CREATE OR REPLACE FUNCTION rate.crawl_job_repeat_failures( + p_min_failures integer DEFAULT 3, + p_since interval DEFAULT interval '3 days', + p_limit integer DEFAULT 50 +) +RETURNS TABLE ( + platform text, + platform_item_id text, + failures bigint, + first_failed_at timestamptz, + last_failed_at timestamptz, + latest_error_kind text +) +LANGUAGE sql +STABLE +SECURITY DEFINER +SET search_path = '' +AS $$ + SELECT g.platform, + pg_catalog.left(g.platform_item_id, 200), + g.failures, + g.first_failed_at, + g.last_failed_at, + pg_catalog.left( + pg_catalog.btrim( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + g.latest_error, + '[a-z][a-z0-9+.-]*://\S+', '', 'gi'), + '"[^"]*("|$)', '"\1', 'g'), + '(?\2', 'gi'), + '\m((set-)?cookie)\s*[=:][^\r\n]*', '\1=', 'gi'), + '[^\s/@:"<>]+:[^\s/@"<>]*@', '@', 'g'), + '[a-z0-9._%+-]+@[a-z0-9-]+(\.[a-z0-9-]+)+', '', 'gi'), + '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', '', 'gi'), + 'eyJ[a-z0-9_-]{8,}(\.[a-z0-9_-]+){0,2}', '', 'gi'), + '[a-z0-9_-]{32,}', '', 'gi'), + '\m[0-9]{1,3}(\.[0-9]{1,3}){3}(:[0-9]+)?', '', 'g'), + '[0-9]{5,}', '#', 'g'), + '\m(bearer|basic)\s+[^\s,;]+', '\1 ', 'gi'), + '\m([a-z_-]*(token|key|secret|password|passwd|signature|authorization|session|credential)[a-z_-]*)\s*[=:]\s*[^\s,;&]+', '\1=', 'gi'), + '[[:space:][:cntrl:]]+', ' ', 'g')), + 200) + FROM ( + SELECT j.platform, + j.platform_item_id, + pg_catalog.count(*)::bigint AS failures, + pg_catalog.min(j.updated_at) AS first_failed_at, + pg_catalog.max(j.updated_at) AS last_failed_at, + (pg_catalog.array_agg(pg_catalog.left(COALESCE(j.last_error, ''), 1000) ORDER BY j.updated_at DESC, j.id DESC))[1] AS latest_error + FROM rate.crawl_jobs j + WHERE j.status IN ('failed', 'skipped') + AND j.platform_item_id IS NOT NULL + AND j.updated_at >= pg_catalog.now() - LEAST(GREATEST(COALESCE(p_since, interval '3 days'), interval '1 hour'), interval '14 days') + AND pg_catalog.left(COALESCE(j.last_error, ''), 1000) ~* '(ECONNRESET|socket hang up|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|ECONNREFUSED|net::ERR_|connection (reset|refused|closed))' + GROUP BY j.platform, j.platform_item_id + HAVING pg_catalog.count(*) >= GREATEST(COALESCE(p_min_failures, 3), 2) + ORDER BY failures DESC, last_failed_at DESC, j.platform, j.platform_item_id + LIMIT LEAST(GREATEST(COALESCE(p_limit, 50), 1), 100) + ) g + ORDER BY g.failures DESC, g.last_failed_at DESC, g.platform, g.platform_item_id; +$$; + +-- Privileges: service_role and rate_worker only. No privilege on crawl_jobs. +REVOKE ALL ON FUNCTION rate.crawl_queue_summary(text[]) FROM PUBLIC, anon, authenticated; +REVOKE ALL ON FUNCTION rate.crawl_job_failures(interval, integer) FROM PUBLIC, anon, authenticated; +REVOKE ALL ON FUNCTION rate.crawl_job_repeat_failures(integer, interval, integer) FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION rate.crawl_queue_summary(text[]) TO service_role, rate_worker; +GRANT EXECUTE ON FUNCTION rate.crawl_job_failures(interval, integer) TO service_role, rate_worker; +GRANT EXECUTE ON FUNCTION rate.crawl_job_repeat_failures(integer, interval, integer) TO service_role, rate_worker; + +COMMIT; diff --git a/supabase/migrations/20261102233000_rate_review_value_score.sql b/supabase/migrations/20261102233000_rate_review_value_score.sql new file mode 100644 index 00000000..62ab08bd --- /dev/null +++ b/supabase/migrations/20261102233000_rate_review_value_score.sql @@ -0,0 +1,451 @@ +-- ============================================================================= +-- Migration: 20261102233000_rate_review_value_score.sql +-- Purpose (DOS/Rate#1547): phase P6 of the DOS/Rate criteria ratings design +-- (DOS/Rate#1531, Docs/Proposals/2026-10-31-criteria-stars-pros-cons.html, section 6; +-- decisions D6 and D7 approved by JOY on 2026-10-31, the P6 apply on 2026-11-02). It needs +-- P2 (20261101120000_rate_criteria_ratings.sql) and P3 +-- (20261102090000_rate_review_submission_v2.sql). Copied unchanged from the reviewed DOS/Rate +-- proposal Docs/Proposals/2026-11-02-dos-me-review-value-score.sql (DOS/Rate#1580) apart from +-- this header. One transaction, then a refresh that is a no-op on the first apply. +-- Verify: scripts/ops/rate-review-value-score-verify.sql (V1 to V7 return 0). +-- Rollback: scripts/ops/rate-review-value-score-rollback.sql. +-- +-- D7: a review's value number lives in the private rate.review_values (no anon or +-- authenticated access); only its tier is public (rate.reviews.value_tier); the writer reads +-- their own number through rate.my_review_values. D6: a review's rating weight becomes +-- LEAST(3, helpfulness_weight x completeness factor), x1.00 to x1.25. No rating changes at +-- apply: the values table starts empty. Details in the proposal's header, below. +-- ============================================================================= +-- +-- ============================================================================= +-- PROPOSAL (DOS/Rate#1547): phase P6 of the DOS/Rate criteria ratings design (DOS/Rate#1531, +-- Docs/Proposals/2026-10-31-criteria-stars-pros-cons.html, section 6; D6 and D7 approved by +-- JOY on 2026-10-31). A DOS-Me migration to review and apply; nothing here is applied by +-- DOS/Rate. Verify: 2026-11-02-dos-me-review-value-score-verify.sql (V1 to V7 return 0). +-- Rollback: 2026-11-02-dos-me-review-value-score-rollback.sql. +-- +-- What it does: +-- rate.review_values a review's value score and breakdown, PRIVATE (D7: the number only for +-- the writer). No anon or authenticated access; service_role and +-- rate_worker read it; only rate.store_review_values writes it. +-- rate.reviews.value_tier the public tier (D7): basic, complete or excellent; NULL when not +-- scored or gated. P2's rate.reviews.value_score and value_breakdown stay +-- empty, held there by a CHECK, so the number never reaches the public +-- table. +-- rate.review_completeness_factor(breakdown) the D6 factor, 1 to 1.25, from the breakdown's +-- weightCompleteness (completeness less the photo of a review without +-- pros and cons, so a marketplace review keeps its weight). Equal to +-- completenessWeightFactor in packages/shared-data/src/rules/review-value.ts. +-- rate.master_rating_extras each review's weight becomes LEAST(3, helpfulness_weight x +-- factor); nothing else in P2's body changes. +-- triggers on rate.review_values a change of a review's factor refreshes its master. +-- rate.store_review_values(p_values) the writer for the scoring job (rate_worker) and +-- service_role: only a breakdown the shared module could produce. +-- rate.my_review_values(p_review_ids) the writer of a review reads its own number and +-- breakdown (authenticated, through rate.review_authors). +-- +-- Depends on P2 (20261101120000) and P3 (20261102090000); both are in production. +-- +-- Rating impact at apply time: none. rate.review_values is new and empty, and a review without +-- a row has factor 1. Ratings move only as the job stores values, each review by at most +25%. +-- +-- Idempotent: IF NOT EXISTS for the table and column, constraints guarded by pg_constraint, +-- CREATE OR REPLACE for functions, DROP ... IF EXISTS before each trigger and policy. Section +-- 7, after the transaction, refreshes masters whose reviews already hold values (a re-apply); +-- on the first apply it changes nothing. +-- ============================================================================= + +BEGIN; + +-- The ALTER TABLEs on rate.reviews take a short ACCESS EXCLUSIVE lock: give up rather than +-- queue every reader behind a long query. Re-run the file if it times out. +SET LOCAL lock_timeout = '5s'; + +-- 1. The public tier, and P2's columns kept empty ----------------------------------------------- +-- First, so rate.reviews goes straight to ACCESS EXCLUSIVE (the CREATE TABLE below would +-- otherwise take SHARE ROW EXCLUSIVE first and upgrade). NOT VALID: no scan under the lock. + +-- A review that already holds a number in P2's public columns could never be updated again +-- under reviews_value_private: refuse the apply instead (move such values out first). One scan +-- of rate.reviews, before any strong lock. +DO $$ +BEGIN + IF to_regclass('rate.review_values') IS NULL + AND EXISTS (SELECT 1 FROM rate.reviews + WHERE value_score IS NOT NULL OR value_breakdown IS NOT NULL) THEN + RAISE EXCEPTION 'rate.reviews has value_score or value_breakdown set; P6 keeps them empty (D7)' + USING ERRCODE = '55000'; + END IF; +END $$; + +ALTER TABLE rate.reviews ADD COLUMN IF NOT EXISTS value_tier text; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conrelid = 'rate.reviews'::regclass + AND conname = 'reviews_value_tier_known') THEN + ALTER TABLE rate.reviews ADD CONSTRAINT reviews_value_tier_known + CHECK (value_tier IS NULL OR value_tier IN ('basic', 'complete', 'excellent')) NOT VALID; + END IF; + -- P2 put the number on the public table; it lives in rate.review_values instead. + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conrelid = 'rate.reviews'::regclass + AND conname = 'reviews_value_private') THEN + ALTER TABLE rate.reviews ADD CONSTRAINT reviews_value_private + CHECK (value_score IS NULL AND value_breakdown IS NULL) NOT VALID; + END IF; +END $$; + +-- 2. The private values ------------------------------------------------------------------------ +-- Writers: only rate.store_review_values (and service_role by hand). The refresh triggers +-- cover INSERT, UPDATE of the breakdown and DELETE; a TRUNCATE or a change of review_id (only +-- service_role can) needs a manual refresh of the masters involved. + +CREATE TABLE IF NOT EXISTS rate.review_values ( + review_id uuid PRIMARY KEY REFERENCES rate.reviews(id) ON DELETE CASCADE, + value_score numeric(5, 2) NOT NULL CHECK (value_score BETWEEN 0 AND 100), + value_breakdown jsonb NOT NULL CHECK (jsonb_typeof(value_breakdown) = 'object'), + updated_at timestamptz NOT NULL DEFAULT now(), + -- Both completeness keys numbers, completeness a step of 5, the weight one equal to it or + -- the 5 of a photo less; the breakdown's score is value_score. + CONSTRAINT review_values_completeness CHECK ( + CASE WHEN jsonb_typeof(value_breakdown -> 'completeness') = 'number' + AND jsonb_typeof(value_breakdown -> 'weightCompleteness') = 'number' + AND jsonb_typeof(value_breakdown -> 'score') = 'number' + THEN (value_breakdown ->> 'completeness')::numeric IN (0, 5, 10, 15, 20, 25, 30, 35) + AND (value_breakdown ->> 'completeness')::numeric + - (value_breakdown ->> 'weightCompleteness')::numeric IN (0, 5) + AND (value_breakdown ->> 'weightCompleteness')::numeric >= 0 + AND round((value_breakdown ->> 'score')::numeric, 2) = value_score + ELSE false END) +); + +COMMENT ON TABLE rate.review_values IS + 'Value score of a review (DOS/Rate#1547). Private (D7): the public sees only rate.reviews.value_tier; the writer reads its own through rate.my_review_values.'; + +ALTER TABLE rate.review_values ENABLE ROW LEVEL SECURITY; +REVOKE ALL ON rate.review_values FROM PUBLIC, anon, authenticated, rate_worker; +GRANT ALL ON rate.review_values TO service_role; +GRANT SELECT ON rate.review_values TO rate_worker; +DROP POLICY IF EXISTS "Service role access review_values" ON rate.review_values; +CREATE POLICY "Service role access review_values" ON rate.review_values + FOR ALL TO service_role USING (true) WITH CHECK (true); +DROP POLICY IF EXISTS "Rate worker read review_values" ON rate.review_values; +CREATE POLICY "Rate worker read review_values" ON rate.review_values + FOR SELECT TO rate_worker USING (true); + + +-- 3. The D6 factor ----------------------------------------------------------------------------- +-- 1 + 0.25 x weightCompleteness / 35 at four decimals. A missing breakdown, a missing or +-- non-number weightCompleteness count as 0 (factor 1); a number outside 0 to 35 as its nearest +-- bound. + +CREATE OR REPLACE FUNCTION rate.review_completeness_factor(p_breakdown jsonb) +RETURNS numeric +LANGUAGE sql +IMMUTABLE +PARALLEL SAFE +SET search_path = '' +AS $$ + SELECT round(1 + 0.25 * LEAST(35, GREATEST(0, + CASE WHEN jsonb_typeof(p_breakdown) = 'object' + AND jsonb_typeof(p_breakdown -> 'weightCompleteness') = 'number' + THEN (p_breakdown ->> 'weightCompleteness')::numeric + ELSE 0 END)) / 35, 4) +$$; + +REVOKE EXECUTE ON FUNCTION rate.review_completeness_factor(jsonb) FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION rate.review_completeness_factor(jsonb) TO service_role; + +-- 4. The rating reads it ----------------------------------------------------------------------- +-- P2's body with two changes, marked P6: the private values joined, and the weight. + +CREATE OR REPLACE FUNCTION rate.master_rating_extras(p_master_id uuid) +RETURNS TABLE (rating_score numeric, rating_weight_sum numeric, criteria_scores jsonb) +LANGUAGE plpgsql +STABLE +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_item uuid := rate.master_rating_item(p_master_id); +BEGIN + RETURN QUERY EXECUTE format($q$ + WITH eligible AS ( + -- P6: the stored weight times the completeness factor (D6), capped at 3. + SELECT LEAST(3.0, r.helpfulness_weight * rate.review_completeness_factor(v.value_breakdown))::numeric AS w, + COALESCE(r.rating_exact, r.rating_star)::numeric AS star, + r.criteria_ratings + FROM rate.products p + JOIN rate.reviews r ON r.product_id = p.id + JOIN rate.review_flags f ON f.review_id = r.id + LEFT JOIN rate.review_values v ON v.review_id = r.id -- P6 + WHERE %s = $1 + AND p.origin = 'crawl' + AND r.origin IN ('crawl', 'community') + AND r.reviewer_type = 'user' + AND COALESCE(f.include_in_rating, true) + AND r.helpfulness_weight > 0 + ), + overall AS ( + SELECT round(COALESCE(sum(w), 0), 2) AS weight_sum, + CASE WHEN sum(w) > 0 THEN round((3 * 4.2 + sum(w * star)) / (3 + sum(w)), 3) END AS score + FROM eligible + WHERE star BETWEEN 1 AND 5 + ), + per_criterion AS ( + SELECT c.key, + count(*) AS writers, + round(sum(e.w), 2) AS weight_sum, + sum(LEAST(1.0, e.w)) AS shown_sum, + round((3 * 4.2 + sum(e.w * (c.value #>> '{}')::numeric)) / (3 + sum(e.w)), 3) AS score + FROM eligible e + CROSS JOIN LATERAL jsonb_each( + CASE WHEN jsonb_typeof(e.criteria_ratings) = 'object' THEN e.criteria_ratings + ELSE '{}'::jsonb END + ) AS c(key, value) + WHERE c.key ~ '^[a-z_]{2,32}$' + AND jsonb_typeof(c.value) = 'number' + AND (c.value #>> '{}')::numeric BETWEEN 1 AND 5 + GROUP BY c.key + ) + SELECT o.score, + o.weight_sum, + COALESCE( + (SELECT jsonb_object_agg(pc.key, jsonb_build_object( + 'score', pc.score, 'weight_sum', pc.weight_sum, 'writers', pc.writers)) + FROM per_criterion pc + WHERE pc.writers >= 3 AND pc.shown_sum >= 1.5), + '{}'::jsonb) + FROM overall o + $q$, CASE WHEN v_item IS NULL THEN 'p.cluster_id' ELSE 'p.item_id' END) + USING COALESCE(v_item, p_master_id); +END $$; + +-- CREATE OR REPLACE keeps P2's grants (service_role only). + +-- A change of a review's factor refreshes its master. Statement-level, as P2's triggers; only +-- rows whose factor changed count, so a rescore with the same completeness refreshes nothing. +CREATE OR REPLACE FUNCTION rate.review_values_refresh_rating_extras() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +BEGIN + IF TG_OP = 'INSERT' THEN + PERFORM rate.refresh_rating_extras_of_products(ARRAY( + SELECT r.product_id FROM changed_new n JOIN rate.reviews r ON r.id = n.review_id + WHERE rate.review_completeness_factor(n.value_breakdown) <> 1)); + ELSIF TG_OP = 'UPDATE' THEN + PERFORM rate.refresh_rating_extras_of_products(ARRAY( + SELECT r.product_id + FROM changed_new n JOIN changed_old o ON o.review_id = n.review_id + JOIN rate.reviews r ON r.id = n.review_id + WHERE rate.review_completeness_factor(n.value_breakdown) + IS DISTINCT FROM rate.review_completeness_factor(o.value_breakdown))); + ELSE + -- A row deleted with its review (ON DELETE CASCADE) finds no review here; the reviews + -- trigger refreshes that master. + PERFORM rate.refresh_rating_extras_of_products(ARRAY( + SELECT r.product_id FROM changed_old o JOIN rate.reviews r ON r.id = o.review_id + WHERE rate.review_completeness_factor(o.value_breakdown) <> 1)); + END IF; + RETURN NULL; +END $$; + +REVOKE EXECUTE ON FUNCTION rate.review_values_refresh_rating_extras() FROM PUBLIC, anon, authenticated; + +DROP TRIGGER IF EXISTS trg_review_values_rating_extras_ins ON rate.review_values; +CREATE TRIGGER trg_review_values_rating_extras_ins AFTER INSERT ON rate.review_values + REFERENCING NEW TABLE AS changed_new + FOR EACH STATEMENT EXECUTE FUNCTION rate.review_values_refresh_rating_extras(); +DROP TRIGGER IF EXISTS trg_review_values_rating_extras_upd ON rate.review_values; +CREATE TRIGGER trg_review_values_rating_extras_upd AFTER UPDATE ON rate.review_values + REFERENCING OLD TABLE AS changed_old NEW TABLE AS changed_new + FOR EACH STATEMENT EXECUTE FUNCTION rate.review_values_refresh_rating_extras(); +DROP TRIGGER IF EXISTS trg_review_values_rating_extras_del ON rate.review_values; +CREATE TRIGGER trg_review_values_rating_extras_del AFTER DELETE ON rate.review_values + REFERENCING OLD TABLE AS changed_old + FOR EACH STATEMENT EXECUTE FUNCTION rate.review_values_refresh_rating_extras(); + +-- 5. The writer -------------------------------------------------------------------------------- +-- p_values: a JSON array of {"id": review uuid, "score": 0-100, "breakdown": {...}}, at most +-- 1000 rows and 2 MB. Each breakdown must be one the shared module could produce: +-- exactly its eleven keys; "version" 1; "gated" a boolean; +-- "completeness" a step of 5 up to 35; "weightCompleteness" 5 less exactly for a photo +-- without pros and cons ("gaps" has "pros_cons" and not "photo"), else equal to it; +-- "specificity" one of the depth points; "verified" 0 or 15; "helpful" 0 to 20; "trust" 0 to +-- 10; "score" the row's score and the parts' sum; +-- "gaps" distinct known gaps; "tier" the module's rule (basic under completeness 30, +-- excellent from score 70, else complete); +-- a gated row: every part 0, tier null, no gaps. +-- A malformed row fails the whole call (22023) and names its index; nothing is written. Rows +-- of reviews that do not exist are skipped. It writes rate.review_values (the number, private) +-- and rate.reviews.value_tier (the tier, public). The reviews are locked FOR NO KEY UPDATE in id +-- order first: two batches never deadlock, and inserts that reference those reviews (flags, +-- lines, votes) are not blocked. Another writer that updates several reviews out of id order +-- can still deadlock with a batch; PostgreSQL aborts one side (40P01), so the job retries. +-- Returns the number of reviews whose stored value changed. Run one job worker at a time; a +-- batch refreshes its masters inside the call. +CREATE OR REPLACE FUNCTION rate.store_review_values(p_values jsonb) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_row jsonb; + v_index integer := 0; + v_b jsonb; + v_score numeric; + v_sum numeric; + v_c numeric; + v_gaps text[]; + v_tier text; + v_changed integer; +BEGIN + IF jsonb_typeof(p_values) IS DISTINCT FROM 'array' THEN + RAISE EXCEPTION 'p_values must be a JSON array' USING ERRCODE = '22023'; + END IF; + IF jsonb_array_length(p_values) > 1000 OR octet_length(p_values::text) > 2097152 THEN + RAISE EXCEPTION 'at most 1000 rows and 2 MB per call' USING ERRCODE = '22023'; + END IF; + FOR v_row IN SELECT value FROM jsonb_array_elements(p_values) LOOP + v_b := v_row -> 'breakdown'; + IF jsonb_typeof(v_row) IS DISTINCT FROM 'object' + OR jsonb_typeof(v_row -> 'id') IS DISTINCT FROM 'string' + OR (v_row ->> 'id') !~ '^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$' + OR jsonb_typeof(v_row -> 'score') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b) IS DISTINCT FROM 'object' + OR v_b - ARRAY['version', 'gated', 'score', 'completeness', 'weightCompleteness', + 'specificity', 'verified', 'helpful', 'trust', 'tier', 'gaps'] <> '{}'::jsonb + OR NOT v_b ?& ARRAY['version', 'gated', 'score', 'completeness', 'weightCompleteness', + 'specificity', 'verified', 'helpful', 'trust', 'tier', 'gaps'] + OR v_b -> 'version' IS DISTINCT FROM '1'::jsonb + OR jsonb_typeof(v_b -> 'gated') IS DISTINCT FROM 'boolean' + OR jsonb_typeof(v_b -> 'score') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b -> 'completeness') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b -> 'weightCompleteness') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b -> 'specificity') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b -> 'verified') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b -> 'helpful') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b -> 'trust') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b -> 'tier') NOT IN ('string', 'null') + OR jsonb_typeof(v_b -> 'gaps') IS DISTINCT FROM 'array' + OR EXISTS (SELECT 1 FROM jsonb_array_elements(v_b -> 'gaps') g + WHERE jsonb_typeof(g) IS DISTINCT FROM 'string') THEN + RAISE EXCEPTION 'row % is not a review value', v_index USING ERRCODE = '22023'; + END IF; + v_score := (v_row ->> 'score')::numeric; + v_c := (v_b ->> 'completeness')::numeric; + v_sum := v_c + (v_b ->> 'specificity')::numeric + + (v_b ->> 'verified')::numeric + (v_b ->> 'helpful')::numeric + + (v_b ->> 'trust')::numeric; + v_gaps := ARRAY(SELECT jsonb_array_elements_text(v_b -> 'gaps')); + v_tier := v_b ->> 'tier'; + IF v_c NOT IN (0, 5, 10, 15, 20, 25, 30, 35) + OR v_c - (v_b ->> 'weightCompleteness')::numeric NOT IN (0, 5) + -- 5 off exactly when a review has a photo, no pros and cons, and some completeness + OR (v_c - (v_b ->> 'weightCompleteness')::numeric = 5) + IS DISTINCT FROM ('pros_cons' = ANY (v_gaps) AND NOT 'photo' = ANY (v_gaps) AND v_c > 0) + OR (v_b ->> 'specificity')::numeric NOT IN (0, 5, 12, 17, 20) + OR (v_b ->> 'verified')::numeric NOT IN (0, 15) + OR (v_b ->> 'helpful')::numeric NOT BETWEEN 0 AND 20 + OR (v_b ->> 'trust')::numeric NOT BETWEEN 0 AND 10 + OR v_score NOT BETWEEN 0 AND 100 + OR (v_b ->> 'score')::numeric <> v_score + OR abs(v_sum - v_score) > 0.01 + OR cardinality(v_gaps) <> (SELECT count(DISTINCT g) FROM unnest(v_gaps) g) + OR NOT v_gaps <@ ARRAY['detail', 'criteria', 'verdict', 'pros_cons', 'criterion_points', + 'photo', 'verified'] + -- in parentheses: plpgsql ends an IF condition at the first bare THEN + OR (CASE WHEN (v_b ->> 'gated')::boolean + THEN v_sum <> 0 OR v_score <> 0 OR v_tier IS NOT NULL OR cardinality(v_gaps) <> 0 + ELSE v_tier IS DISTINCT FROM CASE WHEN v_c < 30 THEN 'basic' + WHEN v_score >= 70 THEN 'excellent' + ELSE 'complete' END + END) THEN + RAISE EXCEPTION 'row % has values out of range', v_index USING ERRCODE = '22023'; + END IF; + v_index := v_index + 1; + END LOOP; + + -- The batch is read straight from p_values in each statement, with no temp table a caller's + -- session could have prepared: a review listed twice, the last row wins; reviews that do not + -- exist are skipped by the joins. + PERFORM 1 FROM rate.reviews r + WHERE r.id IN (SELECT (e ->> 'id')::uuid FROM jsonb_array_elements(p_values) e) + ORDER BY r.id + FOR NO KEY UPDATE; + + INSERT INTO rate.review_values AS rv (review_id, value_score, value_breakdown, updated_at) + SELECT b.review_id, b.value_score, b.value_breakdown, now() + FROM (SELECT DISTINCT ON ((e ->> 'id')::uuid) + (e ->> 'id')::uuid AS review_id, round((e ->> 'score')::numeric, 2) AS value_score, + e -> 'breakdown' AS value_breakdown + FROM jsonb_array_elements(p_values) WITH ORDINALITY AS a(e, n) + ORDER BY (e ->> 'id')::uuid, n DESC) b + JOIN rate.reviews r ON r.id = b.review_id + ORDER BY b.review_id + ON CONFLICT (review_id) DO UPDATE + SET value_score = EXCLUDED.value_score, + value_breakdown = EXCLUDED.value_breakdown, + updated_at = EXCLUDED.updated_at + WHERE (rv.value_score, rv.value_breakdown) + IS DISTINCT FROM (EXCLUDED.value_score, EXCLUDED.value_breakdown); + GET DIAGNOSTICS v_changed = ROW_COUNT; + + UPDATE rate.reviews r + SET value_tier = b.value_breakdown ->> 'tier' + FROM (SELECT DISTINCT ON ((e ->> 'id')::uuid) + (e ->> 'id')::uuid AS review_id, e -> 'breakdown' AS value_breakdown + FROM jsonb_array_elements(p_values) WITH ORDINALITY AS a(e, n) + ORDER BY (e ->> 'id')::uuid, n DESC) b + WHERE r.id = b.review_id + AND r.value_tier IS DISTINCT FROM (b.value_breakdown ->> 'tier'); + + RETURN v_changed; +END $$; + +REVOKE EXECUTE ON FUNCTION rate.store_review_values(jsonb) FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION rate.store_review_values(jsonb) TO service_role, rate_worker; + +-- 6. The writer of a review reads its number --------------------------------------------------- +-- Up to 100 review ids; returns the ones the caller wrote (rate.review_authors) that have a +-- value. Anyone else's reviews, and anon, get nothing. +CREATE OR REPLACE FUNCTION rate.my_review_values(p_review_ids uuid[]) +RETURNS TABLE (review_id uuid, value_score numeric, value_breakdown jsonb) +LANGUAGE plpgsql +STABLE +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_uid uuid := auth.uid(); +BEGIN + IF v_uid IS NULL THEN + RETURN; + END IF; + IF cardinality(p_review_ids) > 100 THEN + RAISE EXCEPTION 'at most 100 review ids per call' USING ERRCODE = '22023'; + END IF; + RETURN QUERY + SELECT v.review_id, v.value_score, v.value_breakdown + FROM rate.review_values v + JOIN rate.review_authors a ON a.review_id = v.review_id + WHERE v.review_id = ANY (p_review_ids) + AND a.author_user_id = v_uid; +END $$; + +REVOKE EXECUTE ON FUNCTION rate.my_review_values(uuid[]) FROM PUBLIC, anon; +GRANT EXECUTE ON FUNCTION rate.my_review_values(uuid[]) TO authenticated, service_role; + +COMMIT; + +-- 7. Masters with values stored before this apply ---------------------------------------------- +-- A no-op on the first apply (the table is new). On a re-apply over kept values it brings their +-- masters back in line. Outside the transaction, so it holds no ALTER TABLE lock. +SELECT rate.refresh_rating_extras_of_products(ARRAY( + SELECT DISTINCT r.product_id + FROM rate.review_values v JOIN rate.reviews r ON r.id = v.review_id + WHERE rate.review_completeness_factor(v.value_breakdown) > 1));