From 119c30a0909712e4001b4c2d57eb1be850f8eb0e Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:06:21 +0700 Subject: [PATCH 1/5] chore(db): mirror the rs#739 and p6 migrations from dos-me (#1587) * chore(db): mirror the rs#739 and p6 migrations from dos-me DOS/DOS.Me#1462 (DOS/Rate-Services#739) and DOS/DOS.Me#1506 (P6) are on production through the promote DOS/DOS.Me#1520 (31001f14). Mirror their migrations byte for byte into supabase/migrations and regenerate packages/shared-data/src/database.types.ts: rate.review_values, rate.reviews.value_tier, store_review_values, my_review_values, review_completeness_factor and the three crawl queue reads. The P6 and P2 proposal replays now start from a stack that has P6: both roll P6 back first (the P6 replay also drops the CHECK its rollback keeps), so they still replay from the stack before it. Nothing is applied by this change. Refs #1547 Co-Authored-By: Claude Opus 5.5 --- packages/shared-data/src/database.types.ts | 80 ++++ scripts/test-criteria-ratings-proposal.mjs | 3 + scripts/test-review-value-proposal.mjs | 6 + ...30140000_rate_worker_crawl_queue_reads.sql | 279 +++++++++++ ...20261102233000_rate_review_value_score.sql | 451 ++++++++++++++++++ 5 files changed, 819 insertions(+) create mode 100644 supabase/migrations/20261030140000_rate_worker_crawl_queue_reads.sql create mode 100644 supabase/migrations/20261102233000_rate_review_value_score.sql diff --git a/packages/shared-data/src/database.types.ts b/packages/shared-data/src/database.types.ts index 32c0907d..a45ea342 100644 --- a/packages/shared-data/src/database.types.ts +++ b/packages/shared-data/src/database.types.ts @@ -943,6 +943,34 @@ export type Database = { }, ]; }; + review_values: { + Row: { + review_id: string; + updated_at: string; + value_breakdown: Json; + value_score: number; + }; + Insert: { + review_id: string; + updated_at?: string; + value_breakdown: Json; + value_score: number; + }; + Update: { + review_id?: string; + updated_at?: string; + value_breakdown?: Json; + value_score?: number; + }; + Relationships: [ + { + foreignKeyName: 'review_values_review_id_fkey'; + columns: ['review_id']; + referencedRelation: 'reviews'; + referencedColumns: ['id']; + }, + ]; + }; review_votes: { Row: { created_at: string; @@ -1006,6 +1034,7 @@ export type Database = { upvotes: number | null; value_breakdown: Json | null; value_score: number | null; + value_tier: string | null; variation: string | null; verdict: string | null; }; @@ -1034,6 +1063,7 @@ export type Database = { upvotes?: number | null; value_breakdown?: Json | null; value_score?: number | null; + value_tier?: string | null; variation?: string | null; verdict?: string | null; }; @@ -1062,6 +1092,7 @@ export type Database = { upvotes?: number | null; value_breakdown?: Json | null; value_score?: number | null; + value_tier?: string | null; variation?: string | null; verdict?: string | null; }; @@ -1177,6 +1208,42 @@ export type Database = { }; Returns: number; }; + crawl_job_failures: { + Args: { p_limit?: number; p_since?: string }; + Returns: { + error_kind: string; + jobs: number; + latest_updated_at: string; + platform: string; + status: string; + type: string; + }[]; + }; + crawl_job_repeat_failures: { + Args: { p_limit?: number; p_min_failures?: number; p_since?: string }; + Returns: { + failures: number; + first_failed_at: string; + last_failed_at: string; + latest_error_kind: string; + platform: string; + platform_item_id: string; + }[]; + }; + crawl_queue_summary: { + Args: { p_platforms?: string[] }; + Returns: { + jobs: number; + newest_updated_at: string; + oldest_created_at: string; + pending_in_backoff: number; + platform: string; + priority: number; + running_lease_expired: number; + status: string; + type: string; + }[]; + }; current_user_can_vote_reviews: { Args: never; Returns: boolean }; dequeue_crawl_job: { Args: { p_lease?: string; p_platform: string; p_worker_id: string }; @@ -1325,6 +1392,14 @@ export type Database = { total_reviews: number; }[]; }; + my_review_values: { + Args: { p_review_ids: string[] }; + Returns: { + review_id: string; + value_breakdown: Json; + value_score: number; + }[]; + }; purge_completed_crawl_jobs: { Args: { p_retention_days?: number }; Returns: number; @@ -1371,6 +1446,10 @@ export type Database = { input_hash: string; }[]; }; + review_completeness_factor: { + Args: { p_breakdown: Json }; + Returns: number; + }; review_input_hash: { Args: { p_author: string; p_comment: string; p_rating: number }; Returns: string; @@ -1487,6 +1566,7 @@ export type Database = { Args: { p_params?: Json; p_source: string }; Returns: string; }; + store_review_values: { Args: { p_values: Json }; Returns: number }; submit_community_review: { Args: { p_author_name: string; diff --git a/scripts/test-criteria-ratings-proposal.mjs b/scripts/test-criteria-ratings-proposal.mjs index 418ccf3c..121ac46a 100644 --- a/scripts/test-criteria-ratings-proposal.mjs +++ b/scripts/test-criteria-ratings-proposal.mjs @@ -54,6 +54,9 @@ const review = (n) => `d0000000-0000-4000-8000-00000000000${n}`; try { const { sql, asRole } = startRateDb({ name: container }); + // The reference stack has P6 (20261102233000), which sits on P2: roll it back first, as + // production would before a P2 rollback. + sql(read('2026-11-02-dos-me-review-value-score-rollback.sql')); const checkAll = (when) => { for (const { id, query } of checks) assert.equal(sql(query), '0', `${id} ${when}`); diff --git a/scripts/test-review-value-proposal.mjs b/scripts/test-review-value-proposal.mjs index 51c66369..cf52967a 100644 --- a/scripts/test-review-value-proposal.mjs +++ b/scripts/test-review-value-proposal.mjs @@ -139,6 +139,12 @@ const expectedScore = (lifted) => { try { const { sql, asRole } = startRateDb({ name: container }); + // The reference stack has P6 (20261102233000): roll it back, with the CHECK the rollback + // keeps, so the proposal replays on the stack before it. + sql(rollback); + sql( + `ALTER TABLE rate.reviews DROP CONSTRAINT IF EXISTS reviews_value_private` + ); const checkAll = (when) => { for (const { id, query } of checks) assert.equal(sql(query), '0', `${id} ${when}`); diff --git a/supabase/migrations/20261030140000_rate_worker_crawl_queue_reads.sql b/supabase/migrations/20261030140000_rate_worker_crawl_queue_reads.sql new file mode 100644 index 00000000..67e23e71 --- /dev/null +++ b/supabase/migrations/20261030140000_rate_worker_crawl_queue_reads.sql @@ -0,0 +1,279 @@ +-- ============================================================================= +-- Migration: 20261030140000_rate_worker_crawl_queue_reads.sql +-- Purpose (DOS/Rate-Services#739, DOS/DOS.Me#1460): read-only aggregates over +-- the crawl queue for the Rate-Services worker role. +-- +-- rate_worker has no grant on rate.crawl_jobs (20261018183000) and must not get +-- one: claim_token fences ack_crawl_job / fail_crawl_job, and payload, progress +-- and target_url hold page data. Its audits only need counts, ages and error +-- groups, so they read the queue through three SECURITY DEFINER functions that +-- return nothing else: +-- +-- rate.crawl_queue_summary(p_platforms) - jobs per (platform, type, +-- status, priority) with the oldest created_at, newest updated_at, pending +-- jobs in a retry backoff and running jobs whose lease ran out; at most 500 +-- rows. p_platforms NULL means every platform. +-- rate.crawl_job_failures(p_since, p_limit) - failed and skipped jobs per +-- (status, platform, type, error kind) updated within p_since; at most 500 +-- rows (p_limit, default 200). +-- rate.crawl_job_repeat_failures(p_min_failures, p_since, p_limit) - listings +-- (platform, public platform_item_id) with at least p_min_failures failed or +-- skipped jobs whose error is a connection error, with the first and last +-- failure and the latest error kind; at most 100 rows (p_limit, default 50). +-- +-- Windows are clamped to 1 hour .. 14 days: purge_completed_crawl_jobs keeps +-- terminal rows 14 days by default (20261018120105), so a longer window would +-- read a partial history. Raise the clamp together with the retention. +-- +-- error_kind is the first 1000 characters of last_error normalized in SQL +-- (each distinct text once) so raw error text never leaves the database: +-- URLs, double- and single-quoted strings (an unclosed quote to the end; +-- b'', r'' and u'' literals included; an apostrophe inside a word such as +-- can't is not a quote), cookie headers, +-- user:password@ credentials, email addresses, UUIDs, JWTs and other 32+ +-- character tokens, IPv4 hosts, numbers of 5+ digits, bearer / basic +-- credentials and key=value secrets are replaced, whitespace and control +-- characters collapse, and the result is cut at 200 characters. Callers still +-- normalize it again (Rate-Services crawlJobErrorKind + redactSensitiveString) +-- and re-sum groups that become equal. platform_item_id is cut at 200 too. +-- +-- Statement timeout: a function-level SET statement_timeout does not bound the +-- statement that calls the function (the timer starts with the outer +-- statement), so none is set here. A PostgREST call runs under the request +-- role's statement_timeout, or authenticator's when that role sets none. +-- +-- EXECUTE goes to service_role and rate_worker only. Nothing is granted on any +-- table, and no data changes. +-- Idempotent: an existing function with another signature or result type is +-- dropped first (CREATE OR REPLACE cannot change a result type), then CREATE OR +-- REPLACE, REVOKE and GRANT. +-- ============================================================================= + +BEGIN; + +-- Fail fast instead of queueing behind long transactions on the shared database. +SET LOCAL lock_timeout = '5s'; + +DO $$ +DECLARE + r RECORD; +BEGIN + FOR r IN + SELECT p.oid::regprocedure AS sig + FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace + LEFT JOIN (VALUES + ('crawl_queue_summary', + 'p_platforms text[]', + 'TABLE(platform text, type text, status text, priority integer, jobs bigint, oldest_created_at timestamp with time zone, newest_updated_at timestamp with time zone, pending_in_backoff bigint, running_lease_expired bigint)'), + ('crawl_job_failures', + 'p_since interval, p_limit integer', + 'TABLE(status text, platform text, type text, error_kind text, jobs bigint, latest_updated_at timestamp with time zone)'), + ('crawl_job_repeat_failures', + 'p_min_failures integer, p_since interval, p_limit integer', + 'TABLE(platform text, platform_item_id text, failures bigint, first_failed_at timestamp with time zone, last_failed_at timestamp with time zone, latest_error_kind text)') + ) AS want(proname, args, result) + ON want.proname = p.proname + AND want.args = pg_get_function_identity_arguments(p.oid) + AND want.result = pg_get_function_result(p.oid) + WHERE n.nspname = 'rate' + AND p.proname IN ('crawl_queue_summary', 'crawl_job_failures', 'crawl_job_repeat_failures') + AND want.proname IS NULL + LOOP + EXECUTE format('DROP FUNCTION %s', r.sig); + END LOOP; +END $$; + +-- A. Queue summary: counts and ages only. +CREATE OR REPLACE FUNCTION rate.crawl_queue_summary(p_platforms text[] DEFAULT NULL) +RETURNS TABLE ( + platform text, + type text, + status text, + priority integer, + jobs bigint, + oldest_created_at timestamptz, + newest_updated_at timestamptz, + pending_in_backoff bigint, + running_lease_expired bigint +) +LANGUAGE sql +STABLE +SECURITY DEFINER +SET search_path = '' +AS $$ + SELECT j.platform, + j.type, + j.status, + j.priority, + pg_catalog.count(*)::bigint, + pg_catalog.min(j.created_at), + pg_catalog.max(j.updated_at), + (pg_catalog.count(*) FILTER (WHERE j.status = 'pending' AND j.backoff_until > pg_catalog.now()))::bigint, + (pg_catalog.count(*) FILTER (WHERE j.status = 'running' AND j.lease_expires < pg_catalog.now()))::bigint + FROM rate.crawl_jobs j + WHERE p_platforms IS NULL OR j.platform = ANY (p_platforms) + GROUP BY j.platform, j.type, j.status, j.priority + ORDER BY j.platform, j.type, j.status, j.priority + LIMIT 500; +$$; + +-- B. Failed and skipped jobs by (status, platform, type, error kind). +-- The error_kind expression is the same in C; keep them in step. +CREATE OR REPLACE FUNCTION rate.crawl_job_failures( + p_since interval DEFAULT interval '3 days', + p_limit integer DEFAULT 200 +) +RETURNS TABLE ( + status text, + platform text, + type text, + error_kind text, + jobs bigint, + latest_updated_at timestamptz +) +LANGUAGE sql +STABLE +SECURITY DEFINER +SET search_path = '' +AS $$ + SELECT f.status, f.platform, f.type, f.error_kind, + pg_catalog.sum(f.n)::bigint AS jobs, + pg_catalog.max(f.latest) AS latest_updated_at + FROM ( + -- Normalize each distinct error once, not once per job. + SELECT r.status, r.platform, r.type, r.n, r.latest, + pg_catalog.left( + pg_catalog.btrim( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + r.raw, + '[a-z][a-z0-9+.-]*://\S+', '', 'gi'), + '"[^"]*("|$)', '"\1', 'g'), + '(?\2', 'gi'), + '\m((set-)?cookie)\s*[=:][^\r\n]*', '\1=', 'gi'), + '[^\s/@:"<>]+:[^\s/@"<>]*@', '@', 'g'), + '[a-z0-9._%+-]+@[a-z0-9-]+(\.[a-z0-9-]+)+', '', 'gi'), + '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', '', 'gi'), + 'eyJ[a-z0-9_-]{8,}(\.[a-z0-9_-]+){0,2}', '', 'gi'), + '[a-z0-9_-]{32,}', '', 'gi'), + '\m[0-9]{1,3}(\.[0-9]{1,3}){3}(:[0-9]+)?', '', 'g'), + '[0-9]{5,}', '#', 'g'), + '\m(bearer|basic)\s+[^\s,;]+', '\1 ', 'gi'), + '\m([a-z_-]*(token|key|secret|password|passwd|signature|authorization|session|credential)[a-z_-]*)\s*[=:]\s*[^\s,;&]+', '\1=', 'gi'), + '[[:space:][:cntrl:]]+', ' ', 'g')), + 200) AS error_kind + FROM ( + SELECT j.status, j.platform, j.type, + pg_catalog.left(COALESCE(j.last_error, ''), 1000) AS raw, + pg_catalog.count(*) AS n, + pg_catalog.max(j.updated_at) AS latest + FROM rate.crawl_jobs j + WHERE j.status IN ('failed', 'skipped') + AND j.updated_at >= pg_catalog.now() - LEAST(GREATEST(COALESCE(p_since, interval '3 days'), interval '1 hour'), interval '14 days') + GROUP BY 1, 2, 3, 4 + ) r + ) f + GROUP BY f.status, f.platform, f.type, f.error_kind + ORDER BY jobs DESC, latest_updated_at DESC, f.status, f.platform, f.type, f.error_kind + LIMIT LEAST(GREATEST(COALESCE(p_limit, 200), 1), 500); +$$; + +-- C. Listings with repeated connection failures: the public marketplace item +-- id, counts, timestamps and the latest error kind (normalized as in B). +CREATE OR REPLACE FUNCTION rate.crawl_job_repeat_failures( + p_min_failures integer DEFAULT 3, + p_since interval DEFAULT interval '3 days', + p_limit integer DEFAULT 50 +) +RETURNS TABLE ( + platform text, + platform_item_id text, + failures bigint, + first_failed_at timestamptz, + last_failed_at timestamptz, + latest_error_kind text +) +LANGUAGE sql +STABLE +SECURITY DEFINER +SET search_path = '' +AS $$ + SELECT g.platform, + pg_catalog.left(g.platform_item_id, 200), + g.failures, + g.first_failed_at, + g.last_failed_at, + pg_catalog.left( + pg_catalog.btrim( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + g.latest_error, + '[a-z][a-z0-9+.-]*://\S+', '', 'gi'), + '"[^"]*("|$)', '"\1', 'g'), + '(?\2', 'gi'), + '\m((set-)?cookie)\s*[=:][^\r\n]*', '\1=', 'gi'), + '[^\s/@:"<>]+:[^\s/@"<>]*@', '@', 'g'), + '[a-z0-9._%+-]+@[a-z0-9-]+(\.[a-z0-9-]+)+', '', 'gi'), + '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', '', 'gi'), + 'eyJ[a-z0-9_-]{8,}(\.[a-z0-9_-]+){0,2}', '', 'gi'), + '[a-z0-9_-]{32,}', '', 'gi'), + '\m[0-9]{1,3}(\.[0-9]{1,3}){3}(:[0-9]+)?', '', 'g'), + '[0-9]{5,}', '#', 'g'), + '\m(bearer|basic)\s+[^\s,;]+', '\1 ', 'gi'), + '\m([a-z_-]*(token|key|secret|password|passwd|signature|authorization|session|credential)[a-z_-]*)\s*[=:]\s*[^\s,;&]+', '\1=', 'gi'), + '[[:space:][:cntrl:]]+', ' ', 'g')), + 200) + FROM ( + SELECT j.platform, + j.platform_item_id, + pg_catalog.count(*)::bigint AS failures, + pg_catalog.min(j.updated_at) AS first_failed_at, + pg_catalog.max(j.updated_at) AS last_failed_at, + (pg_catalog.array_agg(pg_catalog.left(COALESCE(j.last_error, ''), 1000) ORDER BY j.updated_at DESC, j.id DESC))[1] AS latest_error + FROM rate.crawl_jobs j + WHERE j.status IN ('failed', 'skipped') + AND j.platform_item_id IS NOT NULL + AND j.updated_at >= pg_catalog.now() - LEAST(GREATEST(COALESCE(p_since, interval '3 days'), interval '1 hour'), interval '14 days') + AND pg_catalog.left(COALESCE(j.last_error, ''), 1000) ~* '(ECONNRESET|socket hang up|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|ECONNREFUSED|net::ERR_|connection (reset|refused|closed))' + GROUP BY j.platform, j.platform_item_id + HAVING pg_catalog.count(*) >= GREATEST(COALESCE(p_min_failures, 3), 2) + ORDER BY failures DESC, last_failed_at DESC, j.platform, j.platform_item_id + LIMIT LEAST(GREATEST(COALESCE(p_limit, 50), 1), 100) + ) g + ORDER BY g.failures DESC, g.last_failed_at DESC, g.platform, g.platform_item_id; +$$; + +-- Privileges: service_role and rate_worker only. No privilege on crawl_jobs. +REVOKE ALL ON FUNCTION rate.crawl_queue_summary(text[]) FROM PUBLIC, anon, authenticated; +REVOKE ALL ON FUNCTION rate.crawl_job_failures(interval, integer) FROM PUBLIC, anon, authenticated; +REVOKE ALL ON FUNCTION rate.crawl_job_repeat_failures(integer, interval, integer) FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION rate.crawl_queue_summary(text[]) TO service_role, rate_worker; +GRANT EXECUTE ON FUNCTION rate.crawl_job_failures(interval, integer) TO service_role, rate_worker; +GRANT EXECUTE ON FUNCTION rate.crawl_job_repeat_failures(integer, interval, integer) TO service_role, rate_worker; + +COMMIT; diff --git a/supabase/migrations/20261102233000_rate_review_value_score.sql b/supabase/migrations/20261102233000_rate_review_value_score.sql new file mode 100644 index 00000000..62ab08bd --- /dev/null +++ b/supabase/migrations/20261102233000_rate_review_value_score.sql @@ -0,0 +1,451 @@ +-- ============================================================================= +-- Migration: 20261102233000_rate_review_value_score.sql +-- Purpose (DOS/Rate#1547): phase P6 of the DOS/Rate criteria ratings design +-- (DOS/Rate#1531, Docs/Proposals/2026-10-31-criteria-stars-pros-cons.html, section 6; +-- decisions D6 and D7 approved by JOY on 2026-10-31, the P6 apply on 2026-11-02). It needs +-- P2 (20261101120000_rate_criteria_ratings.sql) and P3 +-- (20261102090000_rate_review_submission_v2.sql). Copied unchanged from the reviewed DOS/Rate +-- proposal Docs/Proposals/2026-11-02-dos-me-review-value-score.sql (DOS/Rate#1580) apart from +-- this header. One transaction, then a refresh that is a no-op on the first apply. +-- Verify: scripts/ops/rate-review-value-score-verify.sql (V1 to V7 return 0). +-- Rollback: scripts/ops/rate-review-value-score-rollback.sql. +-- +-- D7: a review's value number lives in the private rate.review_values (no anon or +-- authenticated access); only its tier is public (rate.reviews.value_tier); the writer reads +-- their own number through rate.my_review_values. D6: a review's rating weight becomes +-- LEAST(3, helpfulness_weight x completeness factor), x1.00 to x1.25. No rating changes at +-- apply: the values table starts empty. Details in the proposal's header, below. +-- ============================================================================= +-- +-- ============================================================================= +-- PROPOSAL (DOS/Rate#1547): phase P6 of the DOS/Rate criteria ratings design (DOS/Rate#1531, +-- Docs/Proposals/2026-10-31-criteria-stars-pros-cons.html, section 6; D6 and D7 approved by +-- JOY on 2026-10-31). A DOS-Me migration to review and apply; nothing here is applied by +-- DOS/Rate. Verify: 2026-11-02-dos-me-review-value-score-verify.sql (V1 to V7 return 0). +-- Rollback: 2026-11-02-dos-me-review-value-score-rollback.sql. +-- +-- What it does: +-- rate.review_values a review's value score and breakdown, PRIVATE (D7: the number only for +-- the writer). No anon or authenticated access; service_role and +-- rate_worker read it; only rate.store_review_values writes it. +-- rate.reviews.value_tier the public tier (D7): basic, complete or excellent; NULL when not +-- scored or gated. P2's rate.reviews.value_score and value_breakdown stay +-- empty, held there by a CHECK, so the number never reaches the public +-- table. +-- rate.review_completeness_factor(breakdown) the D6 factor, 1 to 1.25, from the breakdown's +-- weightCompleteness (completeness less the photo of a review without +-- pros and cons, so a marketplace review keeps its weight). Equal to +-- completenessWeightFactor in packages/shared-data/src/rules/review-value.ts. +-- rate.master_rating_extras each review's weight becomes LEAST(3, helpfulness_weight x +-- factor); nothing else in P2's body changes. +-- triggers on rate.review_values a change of a review's factor refreshes its master. +-- rate.store_review_values(p_values) the writer for the scoring job (rate_worker) and +-- service_role: only a breakdown the shared module could produce. +-- rate.my_review_values(p_review_ids) the writer of a review reads its own number and +-- breakdown (authenticated, through rate.review_authors). +-- +-- Depends on P2 (20261101120000) and P3 (20261102090000); both are in production. +-- +-- Rating impact at apply time: none. rate.review_values is new and empty, and a review without +-- a row has factor 1. Ratings move only as the job stores values, each review by at most +25%. +-- +-- Idempotent: IF NOT EXISTS for the table and column, constraints guarded by pg_constraint, +-- CREATE OR REPLACE for functions, DROP ... IF EXISTS before each trigger and policy. Section +-- 7, after the transaction, refreshes masters whose reviews already hold values (a re-apply); +-- on the first apply it changes nothing. +-- ============================================================================= + +BEGIN; + +-- The ALTER TABLEs on rate.reviews take a short ACCESS EXCLUSIVE lock: give up rather than +-- queue every reader behind a long query. Re-run the file if it times out. +SET LOCAL lock_timeout = '5s'; + +-- 1. The public tier, and P2's columns kept empty ----------------------------------------------- +-- First, so rate.reviews goes straight to ACCESS EXCLUSIVE (the CREATE TABLE below would +-- otherwise take SHARE ROW EXCLUSIVE first and upgrade). NOT VALID: no scan under the lock. + +-- A review that already holds a number in P2's public columns could never be updated again +-- under reviews_value_private: refuse the apply instead (move such values out first). One scan +-- of rate.reviews, before any strong lock. +DO $$ +BEGIN + IF to_regclass('rate.review_values') IS NULL + AND EXISTS (SELECT 1 FROM rate.reviews + WHERE value_score IS NOT NULL OR value_breakdown IS NOT NULL) THEN + RAISE EXCEPTION 'rate.reviews has value_score or value_breakdown set; P6 keeps them empty (D7)' + USING ERRCODE = '55000'; + END IF; +END $$; + +ALTER TABLE rate.reviews ADD COLUMN IF NOT EXISTS value_tier text; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conrelid = 'rate.reviews'::regclass + AND conname = 'reviews_value_tier_known') THEN + ALTER TABLE rate.reviews ADD CONSTRAINT reviews_value_tier_known + CHECK (value_tier IS NULL OR value_tier IN ('basic', 'complete', 'excellent')) NOT VALID; + END IF; + -- P2 put the number on the public table; it lives in rate.review_values instead. + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conrelid = 'rate.reviews'::regclass + AND conname = 'reviews_value_private') THEN + ALTER TABLE rate.reviews ADD CONSTRAINT reviews_value_private + CHECK (value_score IS NULL AND value_breakdown IS NULL) NOT VALID; + END IF; +END $$; + +-- 2. The private values ------------------------------------------------------------------------ +-- Writers: only rate.store_review_values (and service_role by hand). The refresh triggers +-- cover INSERT, UPDATE of the breakdown and DELETE; a TRUNCATE or a change of review_id (only +-- service_role can) needs a manual refresh of the masters involved. + +CREATE TABLE IF NOT EXISTS rate.review_values ( + review_id uuid PRIMARY KEY REFERENCES rate.reviews(id) ON DELETE CASCADE, + value_score numeric(5, 2) NOT NULL CHECK (value_score BETWEEN 0 AND 100), + value_breakdown jsonb NOT NULL CHECK (jsonb_typeof(value_breakdown) = 'object'), + updated_at timestamptz NOT NULL DEFAULT now(), + -- Both completeness keys numbers, completeness a step of 5, the weight one equal to it or + -- the 5 of a photo less; the breakdown's score is value_score. + CONSTRAINT review_values_completeness CHECK ( + CASE WHEN jsonb_typeof(value_breakdown -> 'completeness') = 'number' + AND jsonb_typeof(value_breakdown -> 'weightCompleteness') = 'number' + AND jsonb_typeof(value_breakdown -> 'score') = 'number' + THEN (value_breakdown ->> 'completeness')::numeric IN (0, 5, 10, 15, 20, 25, 30, 35) + AND (value_breakdown ->> 'completeness')::numeric + - (value_breakdown ->> 'weightCompleteness')::numeric IN (0, 5) + AND (value_breakdown ->> 'weightCompleteness')::numeric >= 0 + AND round((value_breakdown ->> 'score')::numeric, 2) = value_score + ELSE false END) +); + +COMMENT ON TABLE rate.review_values IS + 'Value score of a review (DOS/Rate#1547). Private (D7): the public sees only rate.reviews.value_tier; the writer reads its own through rate.my_review_values.'; + +ALTER TABLE rate.review_values ENABLE ROW LEVEL SECURITY; +REVOKE ALL ON rate.review_values FROM PUBLIC, anon, authenticated, rate_worker; +GRANT ALL ON rate.review_values TO service_role; +GRANT SELECT ON rate.review_values TO rate_worker; +DROP POLICY IF EXISTS "Service role access review_values" ON rate.review_values; +CREATE POLICY "Service role access review_values" ON rate.review_values + FOR ALL TO service_role USING (true) WITH CHECK (true); +DROP POLICY IF EXISTS "Rate worker read review_values" ON rate.review_values; +CREATE POLICY "Rate worker read review_values" ON rate.review_values + FOR SELECT TO rate_worker USING (true); + + +-- 3. The D6 factor ----------------------------------------------------------------------------- +-- 1 + 0.25 x weightCompleteness / 35 at four decimals. A missing breakdown, a missing or +-- non-number weightCompleteness count as 0 (factor 1); a number outside 0 to 35 as its nearest +-- bound. + +CREATE OR REPLACE FUNCTION rate.review_completeness_factor(p_breakdown jsonb) +RETURNS numeric +LANGUAGE sql +IMMUTABLE +PARALLEL SAFE +SET search_path = '' +AS $$ + SELECT round(1 + 0.25 * LEAST(35, GREATEST(0, + CASE WHEN jsonb_typeof(p_breakdown) = 'object' + AND jsonb_typeof(p_breakdown -> 'weightCompleteness') = 'number' + THEN (p_breakdown ->> 'weightCompleteness')::numeric + ELSE 0 END)) / 35, 4) +$$; + +REVOKE EXECUTE ON FUNCTION rate.review_completeness_factor(jsonb) FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION rate.review_completeness_factor(jsonb) TO service_role; + +-- 4. The rating reads it ----------------------------------------------------------------------- +-- P2's body with two changes, marked P6: the private values joined, and the weight. + +CREATE OR REPLACE FUNCTION rate.master_rating_extras(p_master_id uuid) +RETURNS TABLE (rating_score numeric, rating_weight_sum numeric, criteria_scores jsonb) +LANGUAGE plpgsql +STABLE +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_item uuid := rate.master_rating_item(p_master_id); +BEGIN + RETURN QUERY EXECUTE format($q$ + WITH eligible AS ( + -- P6: the stored weight times the completeness factor (D6), capped at 3. + SELECT LEAST(3.0, r.helpfulness_weight * rate.review_completeness_factor(v.value_breakdown))::numeric AS w, + COALESCE(r.rating_exact, r.rating_star)::numeric AS star, + r.criteria_ratings + FROM rate.products p + JOIN rate.reviews r ON r.product_id = p.id + JOIN rate.review_flags f ON f.review_id = r.id + LEFT JOIN rate.review_values v ON v.review_id = r.id -- P6 + WHERE %s = $1 + AND p.origin = 'crawl' + AND r.origin IN ('crawl', 'community') + AND r.reviewer_type = 'user' + AND COALESCE(f.include_in_rating, true) + AND r.helpfulness_weight > 0 + ), + overall AS ( + SELECT round(COALESCE(sum(w), 0), 2) AS weight_sum, + CASE WHEN sum(w) > 0 THEN round((3 * 4.2 + sum(w * star)) / (3 + sum(w)), 3) END AS score + FROM eligible + WHERE star BETWEEN 1 AND 5 + ), + per_criterion AS ( + SELECT c.key, + count(*) AS writers, + round(sum(e.w), 2) AS weight_sum, + sum(LEAST(1.0, e.w)) AS shown_sum, + round((3 * 4.2 + sum(e.w * (c.value #>> '{}')::numeric)) / (3 + sum(e.w)), 3) AS score + FROM eligible e + CROSS JOIN LATERAL jsonb_each( + CASE WHEN jsonb_typeof(e.criteria_ratings) = 'object' THEN e.criteria_ratings + ELSE '{}'::jsonb END + ) AS c(key, value) + WHERE c.key ~ '^[a-z_]{2,32}$' + AND jsonb_typeof(c.value) = 'number' + AND (c.value #>> '{}')::numeric BETWEEN 1 AND 5 + GROUP BY c.key + ) + SELECT o.score, + o.weight_sum, + COALESCE( + (SELECT jsonb_object_agg(pc.key, jsonb_build_object( + 'score', pc.score, 'weight_sum', pc.weight_sum, 'writers', pc.writers)) + FROM per_criterion pc + WHERE pc.writers >= 3 AND pc.shown_sum >= 1.5), + '{}'::jsonb) + FROM overall o + $q$, CASE WHEN v_item IS NULL THEN 'p.cluster_id' ELSE 'p.item_id' END) + USING COALESCE(v_item, p_master_id); +END $$; + +-- CREATE OR REPLACE keeps P2's grants (service_role only). + +-- A change of a review's factor refreshes its master. Statement-level, as P2's triggers; only +-- rows whose factor changed count, so a rescore with the same completeness refreshes nothing. +CREATE OR REPLACE FUNCTION rate.review_values_refresh_rating_extras() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +BEGIN + IF TG_OP = 'INSERT' THEN + PERFORM rate.refresh_rating_extras_of_products(ARRAY( + SELECT r.product_id FROM changed_new n JOIN rate.reviews r ON r.id = n.review_id + WHERE rate.review_completeness_factor(n.value_breakdown) <> 1)); + ELSIF TG_OP = 'UPDATE' THEN + PERFORM rate.refresh_rating_extras_of_products(ARRAY( + SELECT r.product_id + FROM changed_new n JOIN changed_old o ON o.review_id = n.review_id + JOIN rate.reviews r ON r.id = n.review_id + WHERE rate.review_completeness_factor(n.value_breakdown) + IS DISTINCT FROM rate.review_completeness_factor(o.value_breakdown))); + ELSE + -- A row deleted with its review (ON DELETE CASCADE) finds no review here; the reviews + -- trigger refreshes that master. + PERFORM rate.refresh_rating_extras_of_products(ARRAY( + SELECT r.product_id FROM changed_old o JOIN rate.reviews r ON r.id = o.review_id + WHERE rate.review_completeness_factor(o.value_breakdown) <> 1)); + END IF; + RETURN NULL; +END $$; + +REVOKE EXECUTE ON FUNCTION rate.review_values_refresh_rating_extras() FROM PUBLIC, anon, authenticated; + +DROP TRIGGER IF EXISTS trg_review_values_rating_extras_ins ON rate.review_values; +CREATE TRIGGER trg_review_values_rating_extras_ins AFTER INSERT ON rate.review_values + REFERENCING NEW TABLE AS changed_new + FOR EACH STATEMENT EXECUTE FUNCTION rate.review_values_refresh_rating_extras(); +DROP TRIGGER IF EXISTS trg_review_values_rating_extras_upd ON rate.review_values; +CREATE TRIGGER trg_review_values_rating_extras_upd AFTER UPDATE ON rate.review_values + REFERENCING OLD TABLE AS changed_old NEW TABLE AS changed_new + FOR EACH STATEMENT EXECUTE FUNCTION rate.review_values_refresh_rating_extras(); +DROP TRIGGER IF EXISTS trg_review_values_rating_extras_del ON rate.review_values; +CREATE TRIGGER trg_review_values_rating_extras_del AFTER DELETE ON rate.review_values + REFERENCING OLD TABLE AS changed_old + FOR EACH STATEMENT EXECUTE FUNCTION rate.review_values_refresh_rating_extras(); + +-- 5. The writer -------------------------------------------------------------------------------- +-- p_values: a JSON array of {"id": review uuid, "score": 0-100, "breakdown": {...}}, at most +-- 1000 rows and 2 MB. Each breakdown must be one the shared module could produce: +-- exactly its eleven keys; "version" 1; "gated" a boolean; +-- "completeness" a step of 5 up to 35; "weightCompleteness" 5 less exactly for a photo +-- without pros and cons ("gaps" has "pros_cons" and not "photo"), else equal to it; +-- "specificity" one of the depth points; "verified" 0 or 15; "helpful" 0 to 20; "trust" 0 to +-- 10; "score" the row's score and the parts' sum; +-- "gaps" distinct known gaps; "tier" the module's rule (basic under completeness 30, +-- excellent from score 70, else complete); +-- a gated row: every part 0, tier null, no gaps. +-- A malformed row fails the whole call (22023) and names its index; nothing is written. Rows +-- of reviews that do not exist are skipped. It writes rate.review_values (the number, private) +-- and rate.reviews.value_tier (the tier, public). The reviews are locked FOR NO KEY UPDATE in id +-- order first: two batches never deadlock, and inserts that reference those reviews (flags, +-- lines, votes) are not blocked. Another writer that updates several reviews out of id order +-- can still deadlock with a batch; PostgreSQL aborts one side (40P01), so the job retries. +-- Returns the number of reviews whose stored value changed. Run one job worker at a time; a +-- batch refreshes its masters inside the call. +CREATE OR REPLACE FUNCTION rate.store_review_values(p_values jsonb) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_row jsonb; + v_index integer := 0; + v_b jsonb; + v_score numeric; + v_sum numeric; + v_c numeric; + v_gaps text[]; + v_tier text; + v_changed integer; +BEGIN + IF jsonb_typeof(p_values) IS DISTINCT FROM 'array' THEN + RAISE EXCEPTION 'p_values must be a JSON array' USING ERRCODE = '22023'; + END IF; + IF jsonb_array_length(p_values) > 1000 OR octet_length(p_values::text) > 2097152 THEN + RAISE EXCEPTION 'at most 1000 rows and 2 MB per call' USING ERRCODE = '22023'; + END IF; + FOR v_row IN SELECT value FROM jsonb_array_elements(p_values) LOOP + v_b := v_row -> 'breakdown'; + IF jsonb_typeof(v_row) IS DISTINCT FROM 'object' + OR jsonb_typeof(v_row -> 'id') IS DISTINCT FROM 'string' + OR (v_row ->> 'id') !~ '^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$' + OR jsonb_typeof(v_row -> 'score') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b) IS DISTINCT FROM 'object' + OR v_b - ARRAY['version', 'gated', 'score', 'completeness', 'weightCompleteness', + 'specificity', 'verified', 'helpful', 'trust', 'tier', 'gaps'] <> '{}'::jsonb + OR NOT v_b ?& ARRAY['version', 'gated', 'score', 'completeness', 'weightCompleteness', + 'specificity', 'verified', 'helpful', 'trust', 'tier', 'gaps'] + OR v_b -> 'version' IS DISTINCT FROM '1'::jsonb + OR jsonb_typeof(v_b -> 'gated') IS DISTINCT FROM 'boolean' + OR jsonb_typeof(v_b -> 'score') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b -> 'completeness') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b -> 'weightCompleteness') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b -> 'specificity') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b -> 'verified') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b -> 'helpful') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b -> 'trust') IS DISTINCT FROM 'number' + OR jsonb_typeof(v_b -> 'tier') NOT IN ('string', 'null') + OR jsonb_typeof(v_b -> 'gaps') IS DISTINCT FROM 'array' + OR EXISTS (SELECT 1 FROM jsonb_array_elements(v_b -> 'gaps') g + WHERE jsonb_typeof(g) IS DISTINCT FROM 'string') THEN + RAISE EXCEPTION 'row % is not a review value', v_index USING ERRCODE = '22023'; + END IF; + v_score := (v_row ->> 'score')::numeric; + v_c := (v_b ->> 'completeness')::numeric; + v_sum := v_c + (v_b ->> 'specificity')::numeric + + (v_b ->> 'verified')::numeric + (v_b ->> 'helpful')::numeric + + (v_b ->> 'trust')::numeric; + v_gaps := ARRAY(SELECT jsonb_array_elements_text(v_b -> 'gaps')); + v_tier := v_b ->> 'tier'; + IF v_c NOT IN (0, 5, 10, 15, 20, 25, 30, 35) + OR v_c - (v_b ->> 'weightCompleteness')::numeric NOT IN (0, 5) + -- 5 off exactly when a review has a photo, no pros and cons, and some completeness + OR (v_c - (v_b ->> 'weightCompleteness')::numeric = 5) + IS DISTINCT FROM ('pros_cons' = ANY (v_gaps) AND NOT 'photo' = ANY (v_gaps) AND v_c > 0) + OR (v_b ->> 'specificity')::numeric NOT IN (0, 5, 12, 17, 20) + OR (v_b ->> 'verified')::numeric NOT IN (0, 15) + OR (v_b ->> 'helpful')::numeric NOT BETWEEN 0 AND 20 + OR (v_b ->> 'trust')::numeric NOT BETWEEN 0 AND 10 + OR v_score NOT BETWEEN 0 AND 100 + OR (v_b ->> 'score')::numeric <> v_score + OR abs(v_sum - v_score) > 0.01 + OR cardinality(v_gaps) <> (SELECT count(DISTINCT g) FROM unnest(v_gaps) g) + OR NOT v_gaps <@ ARRAY['detail', 'criteria', 'verdict', 'pros_cons', 'criterion_points', + 'photo', 'verified'] + -- in parentheses: plpgsql ends an IF condition at the first bare THEN + OR (CASE WHEN (v_b ->> 'gated')::boolean + THEN v_sum <> 0 OR v_score <> 0 OR v_tier IS NOT NULL OR cardinality(v_gaps) <> 0 + ELSE v_tier IS DISTINCT FROM CASE WHEN v_c < 30 THEN 'basic' + WHEN v_score >= 70 THEN 'excellent' + ELSE 'complete' END + END) THEN + RAISE EXCEPTION 'row % has values out of range', v_index USING ERRCODE = '22023'; + END IF; + v_index := v_index + 1; + END LOOP; + + -- The batch is read straight from p_values in each statement, with no temp table a caller's + -- session could have prepared: a review listed twice, the last row wins; reviews that do not + -- exist are skipped by the joins. + PERFORM 1 FROM rate.reviews r + WHERE r.id IN (SELECT (e ->> 'id')::uuid FROM jsonb_array_elements(p_values) e) + ORDER BY r.id + FOR NO KEY UPDATE; + + INSERT INTO rate.review_values AS rv (review_id, value_score, value_breakdown, updated_at) + SELECT b.review_id, b.value_score, b.value_breakdown, now() + FROM (SELECT DISTINCT ON ((e ->> 'id')::uuid) + (e ->> 'id')::uuid AS review_id, round((e ->> 'score')::numeric, 2) AS value_score, + e -> 'breakdown' AS value_breakdown + FROM jsonb_array_elements(p_values) WITH ORDINALITY AS a(e, n) + ORDER BY (e ->> 'id')::uuid, n DESC) b + JOIN rate.reviews r ON r.id = b.review_id + ORDER BY b.review_id + ON CONFLICT (review_id) DO UPDATE + SET value_score = EXCLUDED.value_score, + value_breakdown = EXCLUDED.value_breakdown, + updated_at = EXCLUDED.updated_at + WHERE (rv.value_score, rv.value_breakdown) + IS DISTINCT FROM (EXCLUDED.value_score, EXCLUDED.value_breakdown); + GET DIAGNOSTICS v_changed = ROW_COUNT; + + UPDATE rate.reviews r + SET value_tier = b.value_breakdown ->> 'tier' + FROM (SELECT DISTINCT ON ((e ->> 'id')::uuid) + (e ->> 'id')::uuid AS review_id, e -> 'breakdown' AS value_breakdown + FROM jsonb_array_elements(p_values) WITH ORDINALITY AS a(e, n) + ORDER BY (e ->> 'id')::uuid, n DESC) b + WHERE r.id = b.review_id + AND r.value_tier IS DISTINCT FROM (b.value_breakdown ->> 'tier'); + + RETURN v_changed; +END $$; + +REVOKE EXECUTE ON FUNCTION rate.store_review_values(jsonb) FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION rate.store_review_values(jsonb) TO service_role, rate_worker; + +-- 6. The writer of a review reads its number --------------------------------------------------- +-- Up to 100 review ids; returns the ones the caller wrote (rate.review_authors) that have a +-- value. Anyone else's reviews, and anon, get nothing. +CREATE OR REPLACE FUNCTION rate.my_review_values(p_review_ids uuid[]) +RETURNS TABLE (review_id uuid, value_score numeric, value_breakdown jsonb) +LANGUAGE plpgsql +STABLE +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_uid uuid := auth.uid(); +BEGIN + IF v_uid IS NULL THEN + RETURN; + END IF; + IF cardinality(p_review_ids) > 100 THEN + RAISE EXCEPTION 'at most 100 review ids per call' USING ERRCODE = '22023'; + END IF; + RETURN QUERY + SELECT v.review_id, v.value_score, v.value_breakdown + FROM rate.review_values v + JOIN rate.review_authors a ON a.review_id = v.review_id + WHERE v.review_id = ANY (p_review_ids) + AND a.author_user_id = v_uid; +END $$; + +REVOKE EXECUTE ON FUNCTION rate.my_review_values(uuid[]) FROM PUBLIC, anon; +GRANT EXECUTE ON FUNCTION rate.my_review_values(uuid[]) TO authenticated, service_role; + +COMMIT; + +-- 7. Masters with values stored before this apply ---------------------------------------------- +-- A no-op on the first apply (the table is new). On a re-apply over kept values it brings their +-- masters back in line. Outside the transaction, so it holds no ALTER TABLE lock. +SELECT rate.refresh_rating_extras_of_products(ARRAY( + SELECT DISTINCT r.product_id + FROM rate.review_values v JOIN rate.reviews r ON r.id = v.review_id + WHERE rate.review_completeness_factor(v.value_breakdown) > 1)); From 4107a2860f28d17d9d2cd6ba6fbb87b79b669cee Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:23:40 +0700 Subject: [PATCH 2/5] feat(importer): review value scoring job (P6) (#1588) * feat(importer): review value scoring job (P6) apps/importer/src/shopee/jobs/review-values.ts computes reviewValue (@repo/shared-data) for every crawled and community review, with reviewDepth on the whole text, and stores the values that changed through rate.store_review_values as the rate_worker Auth user. Dry run by default, --apply to write, --master to scope one master. Batches of 500 rows and 1.5 MB at most; a deadlock (40P01) is retried four times. After a run it checks every rated master as V5 of the verify file does: rate_worker cannot call rate.master_rating_extras, so the job computes P2's rating with the P6 weight from the rows it read, reads a differing master again, and lists the ones still stale (exit code 2). Refs #1547 * fix(importer): page review value reads, exact v5 check From two fresh-context reviews of the review values job: - Every read pages by key (keyset, 1000 rows, no OFFSET, no page cap), including the `in` lookups on product_id, review_id and cluster_id, which PostgREST cut at 1000 rows: a big master was scored and checked on a subset. - The V5 check computes in exact integers, as numeric does, and compares exactly: D9's 1.5 threshold no longer flips on float sums, and a master one unit off is stale. - Changed values are batched by master, so a backfill refreshes a master once or twice instead of once per batch. - Only a crawled review's purchase mark counts: community reviews took is_verified_buyer's default of true before 20261018100000. - The CLI signs out after a run. Refs #1547 * Merge remote-tracking branch 'origin/dev' into feat/1547-review-values-job Co-Authored-By: Claude Opus 5.5 --- apps/importer/package.json | 5 +- .../shopee/__tests__/review-values.test.ts | 554 ++++++++++++ .../importer/src/shopee/jobs/review-values.ts | 844 ++++++++++++++++++ 3 files changed, 1401 insertions(+), 2 deletions(-) create mode 100644 apps/importer/src/shopee/__tests__/review-values.test.ts create mode 100644 apps/importer/src/shopee/jobs/review-values.ts diff --git a/apps/importer/package.json b/apps/importer/package.json index a37c606d..4885316b 100644 --- a/apps/importer/package.json +++ b/apps/importer/package.json @@ -37,10 +37,11 @@ "lint": "eslint . --max-warnings 0", "format": "prettier --write \"src/**/*.ts\"", "backfill:specs": "tsx src/shopee/jobs/backfill-specs.ts", - "test": "tsx src/shopee/__tests__/open-core-intelligence.test.ts && tsx src/shopee/__tests__/sourced-seeds.test.ts && tsx src/shopee/__tests__/entity-matcher.test.ts && tsx src/shopee/__tests__/item-matcher.test.ts && tsx src/shopee/__tests__/shopee-to-supabase.test.ts && tsx src/shopee/__tests__/paging.test.ts && tsx src/shopee/__tests__/backfill-review-risk.test.ts && tsx src/shopee/__tests__/item-inventory-report.test.ts && tsx src/shopee/__tests__/arena-audit.test.ts && tsx src/shopee/__tests__/item-aware-jobs.test.ts && tsx src/shopee/__tests__/product-points.test.ts && tsx src/shopee/test-ai-enricher.ts && tsx src/shopee/test-review-analyzer.ts", + "test": "tsx src/shopee/__tests__/open-core-intelligence.test.ts && tsx src/shopee/__tests__/sourced-seeds.test.ts && tsx src/shopee/__tests__/entity-matcher.test.ts && tsx src/shopee/__tests__/item-matcher.test.ts && tsx src/shopee/__tests__/shopee-to-supabase.test.ts && tsx src/shopee/__tests__/paging.test.ts && tsx src/shopee/__tests__/backfill-review-risk.test.ts && tsx src/shopee/__tests__/item-inventory-report.test.ts && tsx src/shopee/__tests__/arena-audit.test.ts && tsx src/shopee/__tests__/item-aware-jobs.test.ts && tsx src/shopee/__tests__/product-points.test.ts && tsx src/shopee/__tests__/review-values.test.ts && tsx src/shopee/test-ai-enricher.ts && tsx src/shopee/test-review-analyzer.ts", "test:dist": "node src/shopee/__tests__/dist-exports.smoke.mjs", "backfill:review-risk": "tsx src/shopee/jobs/backfill-review-risk.ts", - "product-points": "tsx src/shopee/jobs/product-points.ts" + "product-points": "tsx src/shopee/jobs/product-points.ts", + "review-values": "tsx src/shopee/jobs/review-values.ts" }, "keywords": [ "product-normalizer", diff --git a/apps/importer/src/shopee/__tests__/review-values.test.ts b/apps/importer/src/shopee/__tests__/review-values.test.ts new file mode 100644 index 00000000..4ea89c6c --- /dev/null +++ b/apps/importer/src/shopee/__tests__/review-values.test.ts @@ -0,0 +1,554 @@ +import assert from 'node:assert/strict'; +import { getRatingCriteria, reviewValue } from '@repo/shared-data/rules'; +import { + BATCH_ROWS, + batchValues, + buildReviewValues, + changedValues, + expectedMasterRatings, + masterRatingMatches, + reviewValueInput, + runReviewValuesJob, + storeBatch, + type ReviewValueWrite, + type ValueListingRow, + type ValuePointRow, + type ValueReviewRow, +} from '../jobs/review-values.js'; +import type { RateSupabaseClient } from '../types.js'; + +const PHONES = 'c0000000-0000-4000-8000-000000000001'; +const M1 = 'a0000000-0000-4000-8000-000000000001'; +const M2 = 'a0000000-0000-4000-8000-000000000002'; +const id = (n: number) => `d0000000-0000-4000-8000-${String(n).padStart(12, '0')}`; + +const review = ( + n: number, + product_id: string, + extra: Partial = {} +): ValueReviewRow => ({ + id: id(n), + product_id, + origin: 'crawl', + reviewer_type: 'user', + helpfulness_weight: 1, + rating_star: 4, + rating_exact: null, + comment: null, + verdict: null, + criteria_ratings: null, + images: null, + is_verified_buyer: null, + upvotes: 0, + downvotes: 0, + value_tier: null, + review_flags: { include_in_rating: true, verdict: 'authentic', flag_category: null }, + ...extra, +}); +const line = ( + n: number, + kind: string, + body: string, + criterion_key: string | null = null, + method = 'writer' +): ValuePointRow => ({ + id: `${id(n)}-${(lineNo += 1)}`, + review_id: id(n), + kind, + body, + criterion_key, + method, +}); +let lineNo = 0; + +const categories = [{ id: PHONES, slug: 'smartphones' }]; +const masters = [ + { id: M1, category_id: PHONES }, + { id: M2, category_id: PHONES }, +]; +const listings: ValueListingRow[] = [ + { id: 'l1', cluster_id: M1, origin: 'crawl' }, + { id: 'l2', cluster_id: M2, origin: 'crawl' }, + { id: 'lseed', cluster_id: M1, origin: 'seed' }, +]; +const allRated = Object.fromEntries(getRatingCriteria('smartphones').map(c => [c.key, 4])); +const reviews = [ + // A marketplace review: a detailed comment, a photo, the purchase mark, votes. + review(1, 'l1', { + rating_star: 5, + comment: + 'Pin dùng hết một ngày rưỡi, màn hình sáng rõ ngoài nắng, camera chụp đêm ổn, máy không nóng khi chơi game lâu.', + images: ['https://cf.shopee.vn/file/a.jpg'], + is_verified_buyer: true, + upvotes: 9, + downvotes: 1, + }), + // A Rate review (form v2): no comment, its text is the verdict and the lines. + review(2, 'l1', { + origin: 'community', + rating_exact: 4, + criteria_ratings: allRated, + verdict: 'Máy mượt, pin dùng hết ngày, đáng tiền trong tầm giá.', + }), + // Excluded by the analyzer; not analyzed yet; filler. + review(3, 'l1', { + comment: 'Pin trâu, màn đẹp, chụp ảnh nét, dùng mượt mà không lag.', + review_flags: { include_in_rating: false, verdict: 'suspicious', flag_category: null }, + }), + review(4, 'l1', { comment: 'Pin trâu, màn đẹp.', review_flags: null }), + review(5, 'l1', { + comment: 'Shop giao nhanh, đánh giá để nhận xu nhé.', + review_flags: { + include_in_rating: true, + verdict: 'authentic', + flag_category: 'low_value_coin_spam', + }, + }), + // Seed data is never scored. + review(6, 'l1', { origin: 'seed', comment: 'Pin trâu.' }), + // An expert review Rate published, not analyzed yet: it is not gated. + review(7, 'l2', { + reviewer_type: 'expert', + helpfulness_weight: 0, + comment: 'Đo pin 5000 mAh được 9 giờ màn hình bật.', + review_flags: { include_in_rating: false, verdict: 'unanalyzed', flag_category: null }, + }), + review(8, 'l2', { rating_star: 3, comment: 'Ổn.' }), +]; +const points = [ + line(2, 'pro', 'Pin dùng cả ngày'), + line(2, 'con', 'Sạc hơi chậm'), + line(2, 'pro', 'Chụp đêm rõ nét', 'camera'), + line(2, 'con', 'Màn hơi tối ngoài nắng', 'display'), + // An AI line from a marketplace comment earns nothing. + line(1, 'pro', 'Pin trâu', null, 'ai'), +]; + +async function run() { + console.log('🧪 Review values job suite (#1547)'); + + // Inputs: gates, depth on the whole text, photo, purchase mark and votes. + const input = (n: number) => + reviewValueInput( + reviews.find(r => r.id === id(n))!, + points.filter(p => p.review_id === id(n)), + 'smartphones' + ); + assert.deepEqual( + [1, 2, 3, 4, 5, 7, 8].map(n => input(n).gated), + [false, false, true, true, true, false, false] + ); + assert.equal(input(1).depth, 'detailed', 'a comment on the product'); + assert.equal(input(7).depth, 'expert_depth', 'a measurement'); + assert.equal(input(2).depth, 'detailed', 'the verdict and the lines are its text'); + assert.equal(input(5).depth, 'filler'); + assert.equal(input(1).hasRealPhoto, true); + assert.equal(input(1).verifiedPurchase, true); + assert.equal( + reviewValueInput({ ...reviews[1]!, is_verified_buyer: true }, [], 'smartphones') + .verifiedPurchase, + false, + "a community review's default purchase mark does not count" + ); + assert.deepEqual([input(1).helpfulVotes, input(1).unhelpfulVotes], [9, 1]); + assert.equal(input(1).authorTrust, null); + assert.equal(input(1).points[0]!.method, 'ai'); + assert.equal(input(2).points.length, 4); + + const values = buildReviewValues({ reviews, points, listings, masters, categories }); + assert.deepEqual( + [...values.keys()], + [1, 2, 3, 4, 5, 7, 8].map(id), + 'seed reviews are not scored' + ); + assert.deepEqual(values.get(id(2)), reviewValue(input(2))); + assert.equal(values.get(id(2))!.completeness, 30, 'everything but a photo'); + assert.equal(values.get(id(1))!.weightCompleteness, 0, 'a photo without pros and cons'); + assert.equal(values.get(id(3))!.score, 0); + assert.equal(values.get(id(3))!.tier, null); + + // Only what changed is stored: the number, the breakdown (key order aside) or the tier. + const v1 = values.get(id(1))!; + const reordered = Object.fromEntries(Object.entries(v1).reverse()); + const stored = [ + { review_id: id(1), value_score: v1.score, value_breakdown: reordered }, + { review_id: id(8), value_score: values.get(id(8))!.score, value_breakdown: values.get(id(8)) }, + { review_id: id(2), value_score: 1, value_breakdown: values.get(id(2)) }, + ]; + const tiers = new Map([ + [id(1), v1.tier], + [id(8), 'excellent'], + ]); + assert.deepEqual( + changedValues(values, stored, tiers).map(r => r.id), + [2, 3, 4, 5, 7, 8].map(id), + 'review 1 is unchanged; 8 has a drifted tier; 2 a different number' + ); + + // Batches: at most BATCH_ROWS rows, and a byte limit. + const many: ReviewValueWrite[] = Array.from({ length: BATCH_ROWS * 2 + 1 }, (_, i) => ({ + id: id(i), + score: v1.score, + breakdown: v1, + })); + assert.deepEqual( + batchValues(many).map(b => b.length), + [BATCH_ROWS, BATCH_ROWS, 1] + ); + const big = { + ...v1, + gaps: Array.from({ length: 20_000 }, () => 'photo'), + } as unknown as typeof v1; + const heavy = Array.from({ length: 40 }, (_, i) => ({ id: id(i), score: 1, breakdown: big })); + const heavyBatches = batchValues(heavy); + assert.ok(heavyBatches.length > 1, 'the byte limit splits a batch'); + assert.ok( + heavyBatches.every(b => Buffer.byteLength(JSON.stringify(b)) <= 1_500_000), + 'every batch fits the byte limit' + ); + + // The rating a master should have (V5): P2's estimate with P6's weight. + const rated = expectedMasterRatings({ + reviews: [ + review(11, 'l1', { rating_star: 5, criteria_ratings: { battery: 4 } }), + review(12, 'l1', { rating_exact: 4, rating_star: 4, criteria_ratings: { battery: 5 } }), + review(13, 'l1', { + rating_star: 2, + helpfulness_weight: 0.4, + criteria_ratings: { battery: 3, 'Bad Key': 5 }, + }), + // Not counted: excluded, no flag, an expert, weight 0, seed, a seed listing. + review(14, 'l1', { + review_flags: { include_in_rating: false, verdict: 'fake', flag_category: null }, + }), + review(15, 'l1', { review_flags: null }), + review(16, 'l1', { reviewer_type: 'expert' }), + review(17, 'l1', { helpfulness_weight: 0 }), + review(18, 'l1', { origin: 'seed' }), + review(19, 'lseed'), + // A NULL include counts. + review(20, 'l2', { + rating_star: 3, + review_flags: { include_in_rating: null, verdict: null, flag_category: null }, + }), + ], + listings, + values: new Map([[id(11), { weightCompleteness: 35 }]]), + }); + // M1: weights 1.25, 1, 0.4: (12.6 + 6.25 + 4 + 0.8) / 5.65 = 4.186. + assert.deepEqual(rated.get(M1), { + rating_score: 4.186, + rating_weight_sum: 2.65, + criteria_scores: { + // (12.6 + 5 + 5 + 1.2) / 5.65; three writers, shown 2.4. + battery: { score: 4.212, weight_sum: 2.65, writers: 3 }, + }, + }); + assert.deepEqual(rated.get(M2), { rating_score: 3.9, rating_weight_sum: 1, criteria_scores: {} }); + // D9's threshold on weights that sum to exactly 1.5 (0.7 + 0.35 + 0.45): floating point + // gives 1.4999999999999998 in this order and would drop the criterion. + const edge = expectedMasterRatings({ + reviews: [0.7, 0.35, 0.45].map((w, i) => + review(30 + i, 'l2', { helpfulness_weight: w, criteria_ratings: { battery: 4 } }) + ), + listings, + values: new Map(), + }); + assert.deepEqual(edge.get(M2)!.criteria_scores, { + battery: { score: 4.133, weight_sum: 1.5, writers: 3 }, + }); + assert.ok( + !masterRatingMatches( + { + id: M2, + rating_score: 4.134, + rating_weight_sum: 1.5, + criteria_scores: edge.get(M2)!.criteria_scores, + }, + edge.get(M2) + ), + 'one unit in the last place is stale, as V5 compares exactly' + ); + assert.ok( + masterRatingMatches( + { + id: M1, + rating_score: '4.186', + rating_weight_sum: '2.65', + criteria_scores: { battery: { score: 4.212, weight_sum: 2.65, writers: 3 } }, + }, + rated.get(M1) + ) + ); + assert.ok( + !masterRatingMatches( + { id: M1, rating_score: 4.186, rating_weight_sum: 2.65, criteria_scores: {} }, + rated.get(M1) + ), + 'a missing criterion is stale' + ); + assert.ok( + !masterRatingMatches( + { id: M2, rating_score: 3.95, rating_weight_sum: 1, criteria_scores: {} }, + rated.get(M2) + ) + ); + assert.ok( + masterRatingMatches( + { id: M2, rating_score: null, rating_weight_sum: 0, criteria_scores: {} }, + undefined + ), + 'a master with nothing counted has no rating' + ); + + // A deadlock is retried; a refusal stops the run. + let calls = 0; + const flaky = { + rpc: () => { + calls += 1; + return Promise.resolve( + calls === 1 + ? { data: null, error: { code: '40P01', message: 'deadlock detected' } } + : { data: 1, error: null } + ); + }, + } as unknown as RateSupabaseClient; + assert.equal(await storeBatch(flaky, many.slice(0, 1), async () => {}), 1); + assert.equal(calls, 2); + const refusing = { + rpc: () => + Promise.resolve({ + data: null, + error: { code: '22023', message: 'row 0 has values out of range' }, + }), + } as unknown as RateSupabaseClient; + await assert.rejects( + storeBatch(refusing, many.slice(0, 1), async () => {}), + /22023/ + ); + let deadlocks = 0; + const stuck = { + rpc: () => { + deadlocks += 1; + return Promise.resolve({ + data: null, + error: { code: '40P01', message: 'deadlock detected' }, + }); + }, + } as unknown as RateSupabaseClient; + await assert.rejects( + storeBatch(stuck, many.slice(0, 1), async () => {}), + /40P01/ + ); + assert.equal(deadlocks, 4, 'four attempts, then the run stops'); + + // The job against a fake client. + const fake = fakeSupabase({ + reviews, + review_points: points, + review_values: [], + products: listings, + master_products: masters.map(m => ({ + ...m, + rating_score: null, + rating_weight_sum: 0, + criteria_scores: {}, + })), + categories, + }); + // The masters as P2 and P6 rate them once the values are stored, except M2, + // which another transaction held during its refresh. + const after = expectedMasterRatings({ reviews, listings, values }); + fake.onStore = () => { + const m1 = fake.tables.master_products!.find(r => r.id === M1)!; + Object.assign(m1, after.get(M1)); + }; + + const dry = await runReviewValuesJob(fake.client); + assert.equal(dry.apply, false); + assert.equal(dry.reviews, 7); + assert.equal(dry.changed, 7); + assert.equal(fake.stores.length, 0, 'a dry run stores nothing'); + assert.deepEqual(dry.staleMasters, [M1, M2], 'the baseline differs before any value is stored'); + + const applied = await runReviewValuesJob(fake.client, { apply: true }); + assert.equal(applied.stored, 7); + assert.equal(fake.stores.length, 1); + assert.deepEqual( + fake.stores[0]!.map(r => r.id), + [1, 2, 3, 4, 5, 7, 8].map(id), + 'one batch, by master then review id' + ); + assert.deepEqual(applied.staleMasters, [M2], 'M2 is reported after a fresh read'); + assert.equal( + fake.tables.reviews!.find(r => r.id === id(2))!.value_tier, + values.get(id(2))!.tier, + 'the tier is public' + ); + + const again = await runReviewValuesJob(fake.client, { apply: true }); + assert.equal(again.changed, 0, 'a second run stores nothing'); + assert.equal(fake.stores.length, 1); + + // One master only. + const one = fakeSupabase({ + reviews, + review_points: points, + review_values: [], + products: listings, + master_products: masters.map(m => ({ + ...m, + rating_score: null, + rating_weight_sum: 0, + criteria_scores: {}, + })), + categories, + }); + const single = await runReviewValuesJob(one.client, { apply: true, masterId: M2 }); + assert.equal(single.reviews, 2); + assert.deepEqual( + one.stores[0]!.map(r => r.id), + [7, 8].map(id) + ); + await assert.rejects(runReviewValuesJob(one.client, { masterId: 'not-a-uuid' }), /UUID/); + + // A big master: 2500 reviews with 2 lines each, read past PostgREST's 1000-row pages, + // both for the whole table and for one master. + const crowd = Array.from({ length: 2500 }, (_, i) => + review(1000 + i, 'l2', { comment: 'Pin trâu, màn hình đẹp, chụp ảnh nét.' }) + ); + const crowdLines = crowd.flatMap(r => [ + { + id: `${r.id}-a`, + review_id: r.id, + kind: 'pro', + body: 'Pin dùng cả ngày', + criterion_key: null, + method: 'writer', + }, + { + id: `${r.id}-b`, + review_id: r.id, + kind: 'con', + body: 'Sạc hơi chậm', + criterion_key: null, + method: 'writer', + }, + ]); + for (const masterId of [null, M2]) { + const bigFake = fakeSupabase({ + reviews: crowd, + review_points: crowdLines, + review_values: [], + products: listings, + master_products: masters.map(m => ({ + ...m, + rating_score: null, + rating_weight_sum: 0, + criteria_scores: {}, + })), + categories, + }); + const big = await runReviewValuesJob(bigFake.client, { apply: true, masterId }); + assert.equal(big.reviews, 2500, `every review read (${masterId ?? 'all'})`); + assert.equal(bigFake.tables.review_values!.length, 2500); + assert.deepEqual( + bigFake.stores.map(b => b.length), + [500, 500, 500, 500, 500] + ); + assert.ok( + bigFake.stores.every(b => b.every(r => r.breakdown.gaps.includes('criteria'))), + 'the lines were read: no review misses its pros and cons' + ); + assert.ok( + bigFake.stores.flat().every(r => !r.breakdown.gaps.includes('pros_cons')), + 'every review has its lines' + ); + } + + console.log( + ' ✅ inputs, changed values, batches, deadlock retry, V5 check, dry run and apply verified' + ); +} + +type Row = Record; + +/** + * A tiny stand-in for the Supabase query builder (select, in, gt, order and + * limit, at most 1000 rows a read as PostgREST) and the writer RPC, which upserts rate.review_values and sets the + * public tier as rate.store_review_values does. + */ +function fakeSupabase(seed: Record) { + const tables: Record = Object.fromEntries( + Object.entries(seed).map(([name, rows]) => [name, rows.map(r => ({ ...(r as Row) }))]) + ); + const stores: ReviewValueWrite[][] = []; + const reads: string[] = []; + const state: { onStore: (() => void) | null } = { onStore: null }; + const copy = (row: Row): Row => ({ ...row }); + const query = (table: string) => { + let rows = [...(tables[table] ?? [])]; + const builder = { + select: () => builder, + order: (key: string) => { + rows.sort((a, b) => (String(a[key]) < String(b[key]) ? -1 : 1)); + return builder; + }, + in: (column: string, values: unknown[]) => { + rows = rows.filter(r => values.includes(r[column])); + return builder; + }, + gt: (column: string, value: string) => { + rows = rows.filter(r => String(r[column]) > value); + return builder; + }, + // PostgREST returns at most 1000 rows, whatever the limit. + limit: (n: number) => { + reads.push(table); + return Promise.resolve({ data: rows.slice(0, Math.min(n, 1000)).map(copy), error: null }); + }, + }; + return builder; + }; + const client = { + from: (table: string) => query(table), + rpc: (name: string, args: { p_values: ReviewValueWrite[] }) => { + assert.equal(name, 'store_review_values'); + stores.push(args.p_values); + let changed = 0; + for (const row of JSON.parse(JSON.stringify(args.p_values)) as ReviewValueWrite[]) { + const values = tables.review_values!; + const old = values.find(v => v.review_id === row.id); + if (old) { + old.value_score = row.score; + old.value_breakdown = row.breakdown; + } else { + values.push({ + review_id: row.id, + value_score: row.score, + value_breakdown: row.breakdown, + }); + } + changed += 1; + const target = tables.reviews!.find(r => r.id === row.id); + if (target) target.value_tier = row.breakdown.tier; + } + state.onStore?.(); + return Promise.resolve({ data: changed, error: null }); + }, + }; + return { + client: client as unknown as RateSupabaseClient, + tables, + stores, + reads, + set onStore(fn: () => void) { + state.onStore = fn; + }, + }; +} + +run().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/apps/importer/src/shopee/jobs/review-values.ts b/apps/importer/src/shopee/jobs/review-values.ts new file mode 100644 index 00000000..b1e688c1 --- /dev/null +++ b/apps/importer/src/shopee/jobs/review-values.ts @@ -0,0 +1,844 @@ +/** + * Review value score writer (DOS/Rate#1547, design #1531 section 6, D6 and D7) + * + * Usage: + * pnpm --filter @repo/crawlers exec tsx src/shopee/jobs/review-values.ts [--apply] [--master ] + * + * Reads every crawled and community review with its flag, its lines + * (rate.review_points), its listing and category, computes reviewValue + * (@repo/shared-data/rules) with the review's depth on its whole text + * (reviewDepth), and stores the values that changed through + * rate.store_review_values. Dry-run by default: it logs how many values it + * would store (counts only: the numbers are private, D7). `--apply` writes, as the rate_worker role: the job signs in as the + * worker's Supabase Auth user (RATE_WORKER_EMAIL and RATE_WORKER_PASSWORD, + * with the project's publishable key), whose access token carries that role. + * It may read the review tables and write values only through the writer, + * which accepts only a breakdown the shared module could produce. + * + * The inputs, as rate.reviews holds them today: + * - gated: the review does not count, as the product points job counts it + * (no flag, excluded, weight 0; an expert review Rate published unless the + * analyzer excluded it), or its depth is filler (6.1); + * - a photo: the review has an image; votes: upvotes and downvotes, as the + * review sort counts them; verified: the marketplace's purchase mark; + * - the writer's trust (axis D) is not stored yet, so it counts as unknown. + * + * Batches hold at most BATCH_ROWS rows and BATCH_BYTES bytes (the writer takes + * 1000 rows and 2 MB). The writer locks a batch's reviews in id order; another + * writer of rate.reviews may not, so a batch a deadlock aborts (40P01) is + * retried. The other side of such a deadlock is usually a crawl writing + * reviews: schedule the job away from the crawl windows. Run it from one + * place at a time (the importer's schedule). Rows are read by keyset pages, + * and every read is held in memory: a few hundred MB per hundred thousand + * reviews with their text and lines. + * + * `stored` counts the review_values rows the writer changed; a row whose + * only change is the public tier counts in `changed`, not in `stored`. + * + * After a run it checks every rated master as V5 of the verify file does + * (scripts/ops/rate-review-value-score-verify.sql in DOS-Me): the stored + * rating, weight sum and criteria scores against what P2's rating with the + * P6 weight gives. rate_worker cannot call rate.master_rating_extras, so the + * job computes it from the rows it read; a master that differs is read again + * on its own before it is reported, since a crawl may have moved it during + * the run. A master still stale was held by another transaction when its + * refresh ran (P2 skips a locked master); service_role refreshes it with + * rate.refresh_master_rating_extras. The CLI exits with code 2 when any is + * left. Masters are scoped by cluster_id: Items (M1+M2) are not in + * production; this check needs their scope once they are. + */ +import { pathToFileURL } from 'node:url'; +import dotenv from 'dotenv'; +import { + reviewDepth, + completenessWeightFactor, + reviewValue, + type ReviewValue, + type ReviewValuePoint, +} from '@repo/shared-data/rules'; +import { resolveBackfillSupabaseTarget } from '../paging.js'; +import { createImporterLogger } from '../logger.js'; +import { scopeSupabaseToRate, type RateSupabaseClient } from '../types.js'; +import { reviewPointWeight, type CategoryRow, type PointMasterRow } from './product-points.js'; + +const logger = createImporterLogger('ReviewValuesJob'); + +/** IDs per `in` filter, so a request URL stays short. */ +const IN_CHUNK = 150; +/** Rows per read; PostgREST returns at most 1000. */ +export const PAGE_ROWS = 1000; +/** Rows per writer call; the writer takes 1000. */ +export const BATCH_ROWS = 500; +/** Bytes per writer call; the writer takes 2 MB. */ +export const BATCH_BYTES = 1_500_000; +/** Attempts per batch when a deadlock (40P01) aborts it. */ +export const DEADLOCK_ATTEMPTS = 4; +/** Stale master ids a log line lists; the result has all of them. */ +const STALE_LOGGED = 50; + +type FlagRow = { + include_in_rating: boolean | null; + verdict: string | null; + flag_category: string | null; +}; + +export interface ValueReviewRow { + id: string; + product_id: string; + origin: string | null; + reviewer_type: string | null; + helpfulness_weight: number | string | null; + rating_star: number | null; + rating_exact: number | string | null; + comment: string | null; + verdict: string | null; + criteria_ratings: unknown; + images: string[] | null; + is_verified_buyer: boolean | null; + upvotes: number | null; + downvotes: number | null; + value_tier: string | null; + review_flags: FlagRow | FlagRow[] | null; +} + +export interface ValuePointRow { + id?: string; + review_id: string; + kind: string; + criterion_key: string | null; + body: string; + method: string; +} + +export interface ValueListingRow { + id: string; + cluster_id: string | null; + origin: string | null; +} + +export interface StoredValueRow { + review_id: string; + value_score: number | string; + value_breakdown: unknown; +} + +export interface MasterRatingRow { + id: string; + rating_score: number | string | null; + rating_weight_sum: number | string | null; + criteria_scores: unknown; +} + +/** One row of rate.store_review_values. */ +export interface ReviewValueWrite { + id: string; + score: number; + breakdown: ReviewValue; +} + +const REVIEW_COLUMNS = + 'id, product_id, origin, reviewer_type, helpfulness_weight, rating_star, rating_exact, ' + + 'comment, verdict, criteria_ratings, images, is_verified_buyer, upvotes, downvotes, ' + + 'value_tier, review_flags(include_in_rating, verdict, flag_category)'; + +/** Reviews the job scores: crawled or community, never seed or test data. */ +const SCORED_REVIEW_ORIGINS = ['crawl', 'community']; + +const flagOf = (review: ValueReviewRow): FlagRow | null => + Array.isArray(review.review_flags) ? (review.review_flags[0] ?? null) : review.review_flags; + +const count = (value: unknown): number => { + const n = Number(value ?? 0); + return Number.isFinite(n) && n > 0 ? n : 0; +}; + +/** The shared module's input for one review. */ +export function reviewValueInput( + review: ValueReviewRow, + lines: readonly ValuePointRow[], + categorySlug: string | null +): Parameters[0] { + const flag = flagOf(review); + const points: ReviewValuePoint[] = lines + .filter(l => l.kind === 'pro' || l.kind === 'con') + .map(l => ({ + kind: l.kind as 'pro' | 'con', + criterionKey: l.criterion_key, + body: l.body, + method: l.method === 'ai' ? 'ai' : 'writer', + })); + const depth = reviewDepth({ + comment: review.comment, + verdict: review.verdict, + points, + reviewerType: review.reviewer_type, + flagCategory: flag?.flag_category ?? null, + }); + const weight = reviewPointWeight({ + id: review.id, + product_id: review.product_id, + origin: review.origin, + reviewer_type: review.reviewer_type, + helpfulness_weight: review.helpfulness_weight, + created_at: null, + platform_created_at: null, + review_flags: flag, + }); + return { + categorySlug, + gated: weight === 0 || depth === 'filler', + depth, + criteriaRatings: review.criteria_ratings, + verdict: review.verdict, + points, + hasRealPhoto: Array.isArray(review.images) && review.images.length > 0, + // A marketplace's mark only: is_verified_buyer defaults to true, and community + // reviews written before 20261018100000 took that default without a purchase. + verifiedPurchase: review.origin === 'crawl' && review.is_verified_buyer === true, + helpfulVotes: count(review.upvotes), + unhelpfulVotes: count(review.downvotes), + authorTrust: null, + }; +} + +/** The category slug of each listing's master, or null. */ +function categoryByListing(input: { + listings: readonly ValueListingRow[]; + masters: readonly PointMasterRow[]; + categories: readonly CategoryRow[]; +}): Map { + const slugByCategory = new Map(input.categories.map(c => [c.id, c.slug])); + const slugByMaster = new Map( + input.masters.map(m => [ + m.id, + m.category_id ? (slugByCategory.get(m.category_id) ?? null) : null, + ]) + ); + return new Map( + input.listings.map(l => [l.id, l.cluster_id ? (slugByMaster.get(l.cluster_id) ?? null) : null]) + ); +} + +/** The value of every review, by review id. */ +export function buildReviewValues(input: { + reviews: readonly ValueReviewRow[]; + points: readonly ValuePointRow[]; + listings: readonly ValueListingRow[]; + masters: readonly PointMasterRow[]; + categories: readonly CategoryRow[]; +}): Map { + const linesByReview = new Map(); + for (const line of input.points) { + const lines = linesByReview.get(line.review_id) ?? []; + lines.push(line); + linesByReview.set(line.review_id, lines); + } + const slugOf = categoryByListing(input); + const out = new Map(); + for (const review of input.reviews) { + if (!SCORED_REVIEW_ORIGINS.includes(review.origin ?? '')) continue; + out.set( + review.id, + reviewValue( + reviewValueInput( + review, + linesByReview.get(review.id) ?? [], + slugOf.get(review.product_id) ?? null + ) + ) + ); + } + return out; +} + +/** JSON with sorted keys, so a stored jsonb and a computed object compare. */ +function canonical(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + if (value && typeof value === 'object') { + const entries = Object.entries(value as Record).sort(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0 + ); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonical(v)}`).join(',')}}`; + } + return JSON.stringify(value ?? null); +} + +/** + * The rows to store: reviews whose stored number, breakdown or public tier + * differs from the computed value. They are ordered by master, then review + * id: each writer call refreshes the masters of its batch, so a master's + * reviews in one or two batches refresh it once or twice, not once per + * batch. The writer still locks each batch in review id order. + */ +export function changedValues( + values: ReadonlyMap, + stored: readonly StoredValueRow[], + tierById: ReadonlyMap, + masterById: ReadonlyMap = new Map() +): ReviewValueWrite[] { + const storedById = new Map(stored.map(row => [row.review_id, row])); + const out: ReviewValueWrite[] = []; + for (const [id, value] of values) { + const row = storedById.get(id); + const same = + row !== undefined && + Number(row.value_score) === value.score && + canonical(row.value_breakdown) === canonical(value) && + (tierById.get(id) ?? null) === value.tier; + if (!same) out.push({ id, score: value.score, breakdown: value }); + } + const order = (row: ReviewValueWrite) => `${masterById.get(row.id) ?? '~'}|${row.id}`; + return out.sort((a, b) => (order(a) < order(b) ? -1 : order(a) > order(b) ? 1 : 0)); +} + +/** Writer calls of at most BATCH_ROWS rows and BATCH_BYTES bytes each. */ +export function batchValues(rows: readonly ReviewValueWrite[]): ReviewValueWrite[][] { + const out: ReviewValueWrite[][] = []; + let batch: ReviewValueWrite[] = []; + let bytes = 2; + for (const row of rows) { + const size = Buffer.byteLength(JSON.stringify(row), 'utf8') + 1; + if (batch.length > 0 && (batch.length >= BATCH_ROWS || bytes + size > BATCH_BYTES)) { + out.push(batch); + batch = []; + bytes = 2; + } + batch.push(row); + bytes += size; + } + if (batch.length > 0) out.push(batch); + return out; +} + +// The rating is computed in exact integers, as PostgreSQL's numeric does: a +// weight in millionths (helpfulness_weight has 2 decimals, the factor 4), a +// star or criterion value in ten-thousandths. Floating point would flip D9's +// `>= 1.5` threshold on sums like 0.7 + 0.35 + 0.45. +const W_UNIT = 1_000_000n; +const V_UNIT = 10_000n; +const PRIOR_SUM = 126n * W_UNIT * V_UNIT; // k x mu = 3 x 4.2 = 12.6, x 10 +const PRIOR_WEIGHT = 3n * W_UNIT; + +const toUnits = (value: number, unit: bigint): bigint => BigInt(Math.round(value * Number(unit))); + +/** round(num / den, digits) half away from zero, for num, den >= 0. */ +const divRound = (num: bigint, den: bigint, digits: number): number => { + const scale = 10n ** BigInt(digits); + return Number((2n * num * scale + den) / (2n * den)) / Number(scale); +}; + +/** (k x mu + sum(w x v)) / (k + sum(w)) at 3 decimals; sumWV carries PRIOR_SUM's /10. */ +const bayes = (sumW: bigint, sumWV: bigint): number => + divRound(PRIOR_SUM + 10n * sumWV, 10n * V_UNIT * (PRIOR_WEIGHT + sumW), 3); + +const weightSum = (sumW: bigint): number => divRound(sumW, W_UNIT, 2); + +/** What rate.master_rating_extras gives for a master, from the rows read. */ +export interface MasterRatingExpected { + rating_score: number | null; + rating_weight_sum: number; + criteria_scores: Record; +} + +const CRITERION_KEY = /^[a-z_]{2,32}$/; + +/** + * P2's rating with P6's weight, per master (cluster_id scope): user reviews + * of crawled listings, crawl or community, with a flag that does not exclude + * them and a weight above 0, each weighing LEAST(3, weight x factor). + */ +export function expectedMasterRatings(input: { + reviews: readonly ValueReviewRow[]; + listings: readonly ValueListingRow[]; + values: ReadonlyMap; +}): Map { + const masterByListing = new Map( + input.listings + .filter(l => l.origin === 'crawl' && typeof l.cluster_id === 'string' && l.cluster_id) + .map(l => [l.id, l.cluster_id as string]) + ); + type Eligible = { w: bigint; star: number; criteria: unknown }; + const byMaster = new Map(); + for (const review of input.reviews) { + const masterId = masterByListing.get(review.product_id); + if (!masterId) continue; + const flag = flagOf(review); + const weight = Number(review.helpfulness_weight ?? 0); + if ( + !SCORED_REVIEW_ORIGINS.includes(review.origin ?? '') || + review.reviewer_type !== 'user' || + flag === null || + flag.include_in_rating === false || + !(weight > 0) + ) { + continue; + } + const exact = review.rating_exact == null ? null : Number(review.rating_exact); + const list = byMaster.get(masterId) ?? []; + const factor = completenessWeightFactor(input.values.get(review.id)?.weightCompleteness); + const w = toUnits(weight, 100n) * toUnits(factor, V_UNIT); + list.push({ + // LEAST(3, helpfulness_weight x factor), as reviewRatingWeight, in millionths. + w: w < 3n * W_UNIT ? w : 3n * W_UNIT, + star: exact ?? Number(review.rating_star), + criteria: review.criteria_ratings, + }); + byMaster.set(masterId, list); + } + const out = new Map(); + for (const [masterId, eligible] of byMaster) { + const rated = eligible.filter(e => e.star >= 1 && e.star <= 5); + const sumW = rated.reduce((s, e) => s + e.w, 0n); + const sumWS = rated.reduce((s, e) => s + e.w * toUnits(e.star, V_UNIT), 0n); + const perKey = new Map(); + for (const e of eligible) { + if (!e.criteria || typeof e.criteria !== 'object' || Array.isArray(e.criteria)) continue; + for (const [key, value] of Object.entries(e.criteria as Record)) { + if (!CRITERION_KEY.test(key) || typeof value !== 'number' || value < 1 || value > 5) { + continue; + } + const acc = perKey.get(key) ?? { writers: 0, w: 0n, shown: 0n, ws: 0n }; + acc.writers += 1; + acc.w += e.w; + acc.shown += e.w < W_UNIT ? e.w : W_UNIT; + acc.ws += e.w * toUnits(value, V_UNIT); + perKey.set(key, acc); + } + } + const criteria: MasterRatingExpected['criteria_scores'] = {}; + for (const [key, acc] of perKey) { + if (acc.writers < 3 || 2n * acc.shown < 3n * W_UNIT) continue; + criteria[key] = { + score: bayes(acc.w, acc.ws), + weight_sum: weightSum(acc.w), + writers: acc.writers, + }; + } + out.set(masterId, { + rating_score: sumW > 0n ? bayes(sumW, sumWS) : null, + rating_weight_sum: weightSum(sumW), + criteria_scores: criteria, + }); + } + return out; +} + +/** Equal at `digits` decimals, as V5 compares the stored numeric exactly. */ +const close = (a: unknown, b: number | null, digits: number): boolean => { + if (a === null || a === undefined || b === null) return (a ?? null) === b; + const scale = 10 ** digits; + return Math.round(Number(a) * scale) === Math.round(b * scale); +}; + +/** Whether a stored master equals what the rating gives (V5 of the verify file). */ +export function masterRatingMatches( + row: MasterRatingRow, + expected: MasterRatingExpected | undefined +): boolean { + const want: MasterRatingExpected = expected ?? { + rating_score: null, + rating_weight_sum: 0, + criteria_scores: {}, + }; + if (!close(row.rating_score, want.rating_score, 3)) return false; + if (!close(row.rating_weight_sum ?? 0, want.rating_weight_sum, 2)) return false; + const stored = + row.criteria_scores && typeof row.criteria_scores === 'object' + ? (row.criteria_scores as Record< + string, + { score?: unknown; weight_sum?: unknown; writers?: unknown } + >) + : {}; + const keys = Object.keys(want.criteria_scores).sort(); + if (canonical(Object.keys(stored).sort()) !== canonical(keys)) return false; + return keys.every(key => { + const s = stored[key] ?? {}; + const w = want.criteria_scores[key]!; + return ( + close(s.score, w.score, 3) && + close(s.weight_sum, w.weight_sum, 2) && + Number(s.writers) === w.writers + ); + }); +} + +function chunks(items: readonly T[], size = IN_CHUNK): T[][] { + const out: T[][] = []; + for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size)); + return out; +} + +type Filter = (query: ReturnType['select']>) => unknown; + +/** + * Every row of a table, or of a filter on it, in pages of PAGE_ROWS keyed on + * `key` (keyset, not OFFSET): each page costs the same however deep it is, + * a row written during the read never shifts the next page, and there is no + * page cap. `columns` must include `key`. + */ +export async function fetchByKey( + supabase: RateSupabaseClient, + table: string, + columns: string, + key: string, + filter: Filter = query => query +): Promise { + const rows: T[] = []; + let last: string | null = null; + for (;;) { + let query = filter(supabase.from(table).select(columns)) as ReturnType< + ReturnType['select'] + >; + if (last !== null) query = query.gt(key, last); + const { data, error } = await query.order(key).limit(PAGE_ROWS); + if (error) throw new Error(`${table}: ${error.message}`); + const page = (data ?? []) as unknown as T[]; + rows.push(...page); + if (page.length < PAGE_ROWS) return rows; + last = String((page[page.length - 1] as Record)[key]); + } +} + +/** The rows whose `column` is one of `ids`, every page of each chunk. */ +async function selectIn( + supabase: RateSupabaseClient, + table: string, + columns: string, + column: string, + ids: readonly string[], + key = 'id' +): Promise { + const rows: T[] = []; + for (const part of chunks([...new Set(ids)])) { + rows.push( + ...(await fetchByKey(supabase, table, columns, key, query => query.in(column, part))) + ); + } + return rows; +} + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +/** Stores one batch, retrying a deadlock; returns the rows the writer changed. */ +export async function storeBatch( + supabase: RateSupabaseClient, + batch: readonly ReviewValueWrite[], + wait: (ms: number) => Promise = sleep +): Promise { + for (let attempt = 1; ; attempt++) { + const { data, error } = await supabase.rpc('store_review_values', { p_values: batch }); + if (!error) return Number(data ?? 0); + if (error.code === '40P01' && attempt < DEADLOCK_ATTEMPTS) { + logger.warn( + 'review_values.deadlock', + `Batch of ${batch.length} hit a deadlock; retrying (${attempt}/${DEADLOCK_ATTEMPTS - 1}).`, + { attempt } + ); + await wait(250 * 2 ** (attempt - 1)); + continue; + } + // 22023: the writer refused a row the module produced, so the module and + // the database disagree (a REVIEW_VALUE_VERSION change needs DOS-Me first). + throw new Error(`store_review_values (${error.code ?? 'no code'}): ${error.message}`); + } +} + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** The rows a master's rating reads, read again on their own. */ +async function loadMasterRating( + supabase: RateSupabaseClient, + masterIds: readonly string[] +): Promise<{ + rows: MasterRatingRow[]; + expected: Map; +}> { + const listings = await selectIn( + supabase, + 'products', + 'id, cluster_id, origin', + 'cluster_id', + masterIds + ); + const reviews = await selectIn( + supabase, + 'reviews', + REVIEW_COLUMNS, + 'product_id', + listings.map(l => l.id) + ); + const stored = await selectIn( + supabase, + 'review_values', + 'review_id, value_score, value_breakdown', + 'review_id', + reviews.map(r => r.id), + 'review_id' + ); + const values = new Map( + stored.map(row => [ + row.review_id, + { + weightCompleteness: (row.value_breakdown as { weightCompleteness?: unknown } | null) + ?.weightCompleteness, + }, + ]) + ); + const rows = await selectIn( + supabase, + 'master_products', + 'id, rating_score, rating_weight_sum, criteria_scores', + 'id', + masterIds + ); + return { rows, expected: expectedMasterRatings({ reviews, listings, values }) }; +} + +export interface ReviewValuesResult { + apply: boolean; + reviews: number; + changed: number; + stored: number; + batches: number; + masters: number; + staleMasters: string[]; +} + +export async function runReviewValuesJob( + supabase: RateSupabaseClient, + options: { + apply?: boolean; + masterId?: string | null; + wait?: (ms: number) => Promise; + } = {} +): Promise { + const apply = options.apply === true; + const only = options.masterId ?? null; + if (only !== null && !UUID.test(only)) throw new Error('--master must be a UUID.'); + + // One master: its listings first, then their reviews. Otherwise every scored review. + const scopedListings = + only === null + ? null + : await selectIn( + supabase, + 'products', + 'id, cluster_id, origin', + 'cluster_id', + [only] + ); + const reviews = + scopedListings === null + ? await fetchByKey(supabase, 'reviews', REVIEW_COLUMNS, 'id', query => + query.in('origin', SCORED_REVIEW_ORIGINS) + ) + : ( + await selectIn( + supabase, + 'reviews', + REVIEW_COLUMNS, + 'product_id', + scopedListings.map(l => l.id) + ) + ).filter(r => SCORED_REVIEW_ORIGINS.includes(r.origin ?? '')); + const reviewIds = reviews.map(r => r.id); + const pointColumns = 'id, review_id, kind, criterion_key, body, method'; + const points = + only === null + ? await fetchByKey(supabase, 'review_points', pointColumns, 'id') + : await selectIn( + supabase, + 'review_points', + pointColumns, + 'review_id', + reviewIds + ); + const valueColumns = 'review_id, value_score, value_breakdown'; + const stored = + only === null + ? await fetchByKey(supabase, 'review_values', valueColumns, 'review_id') + : await selectIn( + supabase, + 'review_values', + valueColumns, + 'review_id', + reviewIds, + 'review_id' + ); + const listings = + scopedListings ?? + (await selectIn( + supabase, + 'products', + 'id, cluster_id, origin', + 'id', + reviews.map(r => r.product_id) + )); + const masterIds = [ + ...new Set(listings.map(l => l.cluster_id).filter((id): id is string => !!id)), + ]; + const masters = await selectIn( + supabase, + 'master_products', + 'id, category_id', + 'id', + masterIds + ); + const categories = await fetchByKey(supabase, 'categories', 'id, slug', 'id'); + + const values = buildReviewValues({ reviews, points, listings, masters, categories }); + const masterOfListing = new Map(listings.map(l => [l.id, l.cluster_id])); + const changed = changedValues( + values, + stored, + new Map(reviews.map(r => [r.id, r.value_tier])), + new Map(reviews.map(r => [r.id, masterOfListing.get(r.product_id) ?? null])) + ); + const batches = batchValues(changed); + logger.info( + 'review_values.plan', + `${apply ? 'Storing' : 'Would store'} ${changed.length} of ${values.size} review values in ${batches.length} batches.`, + { apply, reviews: values.size, changed: changed.length, batches: batches.length } + ); + + let written = 0; + if (apply) { + for (const [index, batch] of batches.entries()) { + written += await storeBatch(supabase, batch, options.wait); + logger.info( + 'review_values.batch', + `Stored batch ${index + 1}/${batches.length} (${batch.length} rows).`, + { batch: index + 1, rows: batch.length } + ); + } + } + + // V5: in a dry run, the values stored now; after --apply, the values just stored. + const effective = new Map(); + for (const row of stored) { + effective.set(row.review_id, { + weightCompleteness: (row.value_breakdown as { weightCompleteness?: unknown } | null) + ?.weightCompleteness, + }); + } + if (apply) for (const row of changed) effective.set(row.id, row.breakdown); + const expected = expectedMasterRatings({ reviews, listings, values: effective }); + const ratedMasters = [...new Set([...expected.keys(), ...masterIds])]; + const current = await selectIn( + supabase, + 'master_products', + 'id, rating_score, rating_weight_sum, criteria_scores', + 'id', + ratedMasters + ); + const suspects = current.filter(row => !masterRatingMatches(row, expected.get(row.id))); + let staleMasters: string[] = []; + if (suspects.length > 0) { + const again = await loadMasterRating( + supabase, + suspects.map(row => row.id) + ); + staleMasters = again.rows + .filter(row => !masterRatingMatches(row, again.expected.get(row.id))) + .map(row => row.id) + .sort(); + } + if (staleMasters.length > 0) { + logger.warn( + 'review_values.stale_masters', + `${staleMasters.length} masters differ from their computed rating (V5); service_role refreshes them with rate.refresh_master_rating_extras.`, + { + masters: staleMasters.slice(0, STALE_LOGGED), + more: Math.max(0, staleMasters.length - STALE_LOGGED), + } + ); + } + + logger.info( + 'review_values.done', + `${apply ? 'Stored' : 'Dry run:'} ${apply ? written : changed.length} review values; ${ratedMasters.length} masters checked, ${staleMasters.length} stale.`, + { + apply, + reviews: values.size, + changed: changed.length, + stored: written, + masters: ratedMasters.length, + stale: staleMasters.length, + } + ); + return { + apply, + reviews: values.size, + changed: changed.length, + stored: written, + batches: batches.length, + masters: ratedMasters.length, + staleMasters, + }; +} + +const isDirectExecution = + typeof process !== 'undefined' && + Array.isArray(process.argv) && + Boolean(process.argv[1]) && + import.meta.url === pathToFileURL(process.argv[1]!).href; + +if (isDirectExecution) { + dotenv.config({ quiet: true }); + dotenv.config({ path: '../../.env', quiet: true }); + (async () => { + const { createClient } = await import('@supabase/supabase-js'); + const rawUrl = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL || ''; + // The publishable key plus the worker user's sign-in: its access token + // carries the rate_worker role, which reads the review tables and writes + // values only through rate.store_review_values. + const key = ( + process.env.SUPABASE_PUBLISHABLE_KEY || + process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY || + '' + ).trim(); + const email = (process.env.RATE_WORKER_EMAIL || '').trim(); + const password = process.env.RATE_WORKER_PASSWORD || ''; + if (!rawUrl.trim() || !key || !email || !password) { + throw new Error( + 'Missing SUPABASE_URL, SUPABASE_PUBLISHABLE_KEY, RATE_WORKER_EMAIL or RATE_WORKER_PASSWORD in environment.' + ); + } + const { url, host } = resolveBackfillSupabaseTarget(rawUrl); + const client = createClient(url, key, { auth: { persistSession: false } }); + const { data: auth, error: authError } = await client.auth.signInWithPassword({ + email, + password, + }); + if (authError || !auth.session) { + throw new Error( + `Sign-in as the rate_worker user failed: ${authError?.message ?? 'no session'}` + ); + } + const role = (auth.user?.role ?? '').trim(); + if (role !== 'rate_worker') { + throw new Error(`The signed-in user has role '${role}', not rate_worker; refusing to run.`); + } + const supabase = scopeSupabaseToRate(client); + const apply = process.argv.includes('--apply'); + const at = process.argv.indexOf('--master'); + const masterId = at >= 0 ? (process.argv[at + 1] ?? '') : null; + // The host is not secret; it shows which project the job reads. + logger.info('review_values.target', `Target: ${host} (${apply ? 'apply' : 'dry run'})`, { + host, + }); + try { + const result = await runReviewValuesJob(supabase, { apply, masterId }); + if (result.staleMasters.length > 0) process.exitCode = 2; + } finally { + // One run, one session: do not leave its refresh token behind. + await client.auth.signOut().catch(() => undefined); + } + })().catch((err: unknown) => { + logger.error( + 'review_values.fatal', + `Review values job failed: ${err instanceof Error ? err.message : String(err)}` + ); + process.exit(1); + }); +} From 9864158710b18cd99fbd59cc3f0192e82e3a70df Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:08:27 +0700 Subject: [PATCH 3/5] feat(web,importer): v2 pros and cons and the Shopee analyzer use reviewDepth (#1589) * feat(web,importer): v2 pros and cons and the Shopee analyzer use reviewDepth (#1583) - The web review reads embed review_points, and the trust weight passes the v2 verdict line and the writer's pros and cons to reviewDepth with the comment; AI-extracted lines stay out. - The Shopee analyzer's axis C is reviewDepth (coin spam as filler), so the web, the analyzer and the review value job classify depth alike. Closes #1583 * test(web): v2 trust depth tests in their own file, with points-only and dedupe cases (#1583 review) The added block pushed core-rate-architecture past tsc's control-flow limit. The new file checks writer lines alone and that a verdict said twice weighs the same; a stale comment on the importer's product regex is fixed. Refs #1583 Co-Authored-By: Claude Opus 5.5 --- .../importer/src/shopee/ai-review-analyzer.ts | 55 +++----------- .../src/shopee/test-review-analyzer.ts | 36 ++++++++- apps/web/package.json | 2 +- .../src/lib/__tests__/trust-depth-v2.test.ts | 75 +++++++++++++++++++ apps/web/src/lib/rate-api.ts | 38 ++++++++-- apps/web/src/lib/rate-domain-core.ts | 12 ++- 6 files changed, 165 insertions(+), 53 deletions(-) create mode 100644 apps/web/src/lib/__tests__/trust-depth-v2.test.ts diff --git a/apps/importer/src/shopee/ai-review-analyzer.ts b/apps/importer/src/shopee/ai-review-analyzer.ts index e1bac693..1f750546 100644 --- a/apps/importer/src/shopee/ai-review-analyzer.ts +++ b/apps/importer/src/shopee/ai-review-analyzer.ts @@ -1,6 +1,6 @@ import crypto from 'node:crypto'; import { z, MAX_VALID_VND_PRICE, seedingRiskPenalty } from '@repo/shared-data'; -import { scopeLlmReviewSummary } from '@repo/shared-data/rules'; +import { reviewDepth, scopeLlmReviewSummary, type ReviewDepth } from '@repo/shared-data/rules'; import { resolveValidHttpBaseUrl, extractJsonObjectString, @@ -80,14 +80,8 @@ export type Independence = | 'undisclosed_paid' | 'conflict_of_interest'; -export type Depth = - | 'stars_only' - | 'platform_label' - | 'filler' - | 'low_effort' - | 'specific_short' - | 'detailed' - | 'expert_depth'; +/** Axis C, as `reviewDepth` in @repo/shared-data/rules classifies it. */ +export type Depth = ReviewDepth; export type SubjectKind = 'master_product' | 'listing' | 'brand' | 'merchant' | 'reviewer' | 'expert' | 'review'; @@ -680,16 +674,12 @@ export function isRatingOnlyComment(comment: string | null | undefined): boolean const SHIPPING_ASPECT_RE = /(?:giao\s*hàng|ship(?:per)?|vận\s*chuyển|đóng\s*gói|bọc\s*chống\s*sốc|hộp\s*(?:bị\s*)?(?:móp|rách|nát)|giao\s*(?:nhanh|chậm|trễ|lâu)|nhân\s*viên\s*giao|đơn\s*vị\s*vận\s*chuyển)/i; -// Whole words only (Unicode-aware), so "hư" in "như" or "pin" in "shopping" -// do not count as a product aspect (DOS/Rate#601). -const PRODUCT_ASPECT_RE = - /(? = {}; @@ -968,30 +954,13 @@ export function assessReview6Axes( independence = 'affiliate'; } - // 4. Axis C: Depth - let depth: Depth; - if (cleanText.length === 0) { - depth = 'stars_only'; - } else if (RATING_LABEL_RE.test(cleanText)) { - depth = 'platform_label'; - } else if (options.isCoinSpam) { - depth = 'filler'; - } else if ( - safeReview.reviewer_type === 'expert' || - TECHNICAL_MEASUREMENT_RE.test(comment) || - (comment.length >= 240 && productAspectCount >= 2) - ) { - depth = 'expert_depth'; - } else if ( - (comment.length >= 90 && (productAspectCount >= 1 || PRODUCT_ASPECT_RE.test(comment))) || - productAspectCount >= 2 - ) { - depth = 'detailed'; - } else if (productAspectCount >= 1 || PRODUCT_ASPECT_RE.test(comment)) { - depth = comment.length >= 30 ? 'specific_short' : 'low_effort'; - } else { - depth = 'low_effort'; - } + // 4. Axis C: Depth, the classifier the web and the review value job use + // (@repo/shared-data/rules, DOS/Rate#1583). Coin spam is filler. + const depth: Depth = reviewDepth({ + comment, + reviewerType: safeReview.reviewer_type, + flagCategory: options.isCoinSpam ? 'low_value_coin_spam' : null, + }); const hasRealMedia = Array.isArray(safeReview.images) && diff --git a/apps/importer/src/shopee/test-review-analyzer.ts b/apps/importer/src/shopee/test-review-analyzer.ts index 50639024..3dd22478 100644 --- a/apps/importer/src/shopee/test-review-analyzer.ts +++ b/apps/importer/src/shopee/test-review-analyzer.ts @@ -4,7 +4,7 @@ */ import http from 'node:http'; import assert from 'node:assert/strict'; -import { LLM_REVIEW_SUMMARY_PREFIX } from '@repo/shared-data/rules'; +import { LLM_REVIEW_SUMMARY_PREFIX, reviewDepth } from '@repo/shared-data/rules'; import { readFileSync } from 'node:fs'; import { AiReviewAnalyzer, @@ -2934,6 +2934,40 @@ async function run() { ); } + // The analyzer's axis C is the shared reviewDepth, as on the web and in the + // review value job (DOS/Rate#1583). + { + for (const comment of [ + '', + 'Hài lòng', + 'Tốt', + 'Pin trâu', + 'Pin trâu, camera đẹp', + 'Pin dùng được cả ngày, màn hình đẹp.', + 'Pin 5000mAh dùng cả ngày', + 'Pin dùng được cả ngày với nhu cầu lướt web, màn hình sáng rõ ngoài trời. '.repeat(2), + Array(40).fill('pin ok').join(' '), + ]) { + assert.equal( + assessReview6Axes({ id: 'd', rating_star: 5, comment }).depth, + reviewDepth({ comment }), + 'depth of ' + JSON.stringify(comment.slice(0, 40)) + ); + } + assert.equal( + assessReview6Axes({ id: 'd', rating_star: 5, comment: 'Pin trâu' }, { isCoinSpam: true }) + .depth, + 'filler' + ); + assert.equal( + assessReview6Axes({ id: 'd', rating_star: 5, comment: 'ok', reviewer_type: 'expert' }).depth, + 'expert_depth' + ); + console.log( + '\u2705 [Axis C] the analyzer classifies depth with the shared reviewDepth (#1583)' + ); + } + console.log('\n🎉 ALL AI REVIEW ANALYZER TESTS PASSED!'); } diff --git a/apps/web/package.json b/apps/web/package.json index f6cb17c3..ced10a3b 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -8,7 +8,7 @@ "lint": "eslint . --max-warnings 0", "preview": "next build && next start", "start": "next start", - "test": "pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/core-rate-architecture.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-model.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/write-paths.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/spec-scope.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/item-fields.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/logger.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-form-v2.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/concurrency.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-profile-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-review-risk.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/watchlist-sizes.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-compare.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-variant-links.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/compare-params.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/catalog-listing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/discount-verdict-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-detail-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-competitors.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/analyzer-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sold-count.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/content-lang.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/image-url.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/client-messages.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/seo.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/offer-plausibility.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-market-stats.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sitemap-lastmod.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-badge-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-criteria.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/rating-score.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/rating-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/focus-trap.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/display-truth.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/spec-template.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-spec-rows.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sale-state.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/csp.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/device-images.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/render-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/model-match.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/developers-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/plural-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/locale-format.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/paged-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/server-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/hero-products.test.ts", + "test": "pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/core-rate-architecture.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-model.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/write-paths.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/spec-scope.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/item-fields.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/logger.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-form-v2.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/concurrency.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-profile-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-review-risk.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/watchlist-sizes.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-compare.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-variant-links.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/compare-params.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/catalog-listing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/discount-verdict-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-detail-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-competitors.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/analyzer-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sold-count.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/content-lang.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/image-url.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/client-messages.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/seo.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/offer-plausibility.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-market-stats.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sitemap-lastmod.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-badge-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-criteria.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/trust-depth-v2.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/rating-score.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/rating-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/focus-trap.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/display-truth.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/spec-template.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-spec-rows.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sale-state.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/csp.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/device-images.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/render-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/model-match.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/developers-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/plural-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/locale-format.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/paged-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/server-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/hero-products.test.ts", "typecheck": "tsc --noEmit" }, "browserslist": [ diff --git a/apps/web/src/lib/__tests__/trust-depth-v2.test.ts b/apps/web/src/lib/__tests__/trust-depth-v2.test.ts new file mode 100644 index 00000000..82a6e22e --- /dev/null +++ b/apps/web/src/lib/__tests__/trust-depth-v2.test.ts @@ -0,0 +1,75 @@ +import assert from 'node:assert/strict'; +import { evaluateReviewTrustAssessment } from '../rate-domain'; + +// A form v2 review: its verdict and its writer's pros and cons count with +// the comment for axis C; AI-extracted lines do not (DOS/Rate#1583). +const v2Review = { + rating_star: 5, + comment: '', + reviewer_type: 'user' as const, + upvotes: 0, + downvotes: 0, + report_count: 0, + flag_category: 'authentic' as const, + verdict: 'authentic' as const, + include_in_rating: true, + signals: null, +}; +const v2Text = { + verdict: 'Máy mượt, đáng tiền trong tầm giá.', + points: [ + { + kind: 'pro', + criterion_key: null, + body: 'Pin dùng cả ngày', + method: 'writer', + }, + { + kind: 'con', + criterion_key: 'battery', + body: 'Sạc hơi chậm', + method: 'writer', + }, + ], +}; +const v2Bare = evaluateReviewTrustAssessment(v2Review); +const v2Eval = evaluateReviewTrustAssessment(v2Review, v2Text); +assert.equal(v2Bare.depth, 'stars_only'); +assert.equal(v2Eval.depth, 'specific_short'); +assert.ok(v2Eval.breakdown.weight > v2Bare.breakdown.weight); +assert.equal( + evaluateReviewTrustAssessment(v2Review, { + points: v2Text.points.map((p) => ({ ...p, method: 'ai' })), + }).depth, + 'stars_only' +); +// The writer's lines count on their own: without a verdict, two lines +// that name the product reach specific_short. +assert.equal( + evaluateReviewTrustAssessment(v2Review, { + points: [ + { kind: 'pro', body: 'Pin dùng được cả ngày', method: 'writer' }, + { kind: 'con', body: 'Sạc hơi chậm nhưng ổn', method: 'writer' }, + ], + }).depth, + 'specific_short' +); +// v2 stores the verdict as the comment when there is no free text: said +// twice, it counts once. Doubled, this 53-character verdict would reach +// the 90 of `detailed`. +const longVerdict = 'Máy mượt, pin dùng hết ngày, đáng tiền trong tầm giá.'; +const once = evaluateReviewTrustAssessment({ + ...v2Review, + comment: longVerdict, +}); +const twice = evaluateReviewTrustAssessment( + { ...v2Review, comment: longVerdict }, + { verdict: longVerdict, points: null } +); +assert.equal(once.depth, 'specific_short'); +assert.equal(twice.depth, 'specific_short'); +assert.equal(twice.breakdown.weight, once.breakdown.weight); + +console.log( + 'trust-depth-v2: the verdict and pros and cons of a v2 review count for axis C' +); diff --git a/apps/web/src/lib/rate-api.ts b/apps/web/src/lib/rate-api.ts index 113cb708..014b80ef 100644 --- a/apps/web/src/lib/rate-api.ts +++ b/apps/web/src/lib/rate-api.ts @@ -1688,6 +1688,20 @@ interface RawReviewFlagRow { signals?: Record | null; } +/** + * A review row with its analyzer flag and the writer's pros and cons + * (form v2), which axis C reads with the comment (DOS/Rate#1583). + */ +const REVIEW_SELECT = + '*, review_flags(*), review_points(kind, criterion_key, body, method)'; + +interface RawReviewPointRow { + kind?: string | null; + criterion_key?: string | null; + body?: string | null; + method?: string | null; +} + interface RawReviewRow { id: string; author_name?: string | null; @@ -1702,6 +1716,9 @@ interface RawReviewRow { helpfulness_weight?: number | string | null; criteria_ratings?: Record | null; review_flags?: RawReviewFlagRow | RawReviewFlagRow[] | null; + /** The form v2 verdict line; the analyzer's verdict is in review_flags. */ + verdict?: string | null; + review_points?: RawReviewPointRow[] | null; } /** @@ -1820,7 +1837,7 @@ export async function getMasterProductReviews( (chunk) => supabase .from('reviews') - .select('*, review_flags(*)') + .select(REVIEW_SELECT) .in('product_id', chunk) .order('created_at', { ascending: false }) .limit(50) @@ -1891,7 +1908,10 @@ export async function getMasterProductReviews( reason: vf?.reason || null, signals: vf?.signals || null, }; - item.trust_assessment = evaluateReviewTrustAssessment(item); + item.trust_assessment = evaluateReviewTrustAssessment(item, { + verdict: r.verdict, + points: r.review_points, + }); return item; }); } @@ -3717,7 +3737,7 @@ export async function getMerchantProfileBase( (chunk) => supabase .from('reviews') - .select('*, review_flags(*)') + .select(REVIEW_SELECT) .in('product_id', chunk) .order('created_at', { ascending: false }) .limit(60) @@ -3888,7 +3908,10 @@ export async function getMerchantProfileBase( reason: vf?.reason || null, signals: vf?.signals || null, }; - item.trust_assessment = evaluateReviewTrustAssessment(item); + item.trust_assessment = evaluateReviewTrustAssessment(item, { + verdict: r.verdict, + points: r.review_points, + }); return item; }); @@ -4134,7 +4157,7 @@ export async function getRatePlatformStats(): Promise { supabase.from('reviews').select('id', { count: 'exact', head: true }), supabase .from('reviews') - .select('*, review_flags(*)') + .select(REVIEW_SELECT) .order('created_at', { ascending: false }) .limit(60), // Platform-wide review verdict counts (DOS/Rate#455), with the same rules @@ -4327,7 +4350,10 @@ export async function getRatePlatformStats(): Promise { masterSlug: pInfo?.masterSlug || null, masterRouteHref: pInfo?.masterRouteHref || null, }; - item.trust_assessment = evaluateReviewTrustAssessment(item); + item.trust_assessment = evaluateReviewTrustAssessment(item, { + verdict: r.verdict, + points: r.review_points, + }); return item; }); diff --git a/apps/web/src/lib/rate-domain-core.ts b/apps/web/src/lib/rate-domain-core.ts index c06108d7..e97cee70 100644 --- a/apps/web/src/lib/rate-domain-core.ts +++ b/apps/web/src/lib/rate-domain-core.ts @@ -204,7 +204,7 @@ export function computeWilsonLowerBound( const WEB_SHIPPING_ASPECT_RE = /(?:giao\s*hàng|ship(?:per)?|vận\s*chuyển|đóng\s*gói|bọc\s*hàng|hộp\s*móp|giao\s*nhanh|giao\s*chậm|tư\s*vấn|chăm\s*sóc\s*khách)/i; -// Any mention of the product or of using it, based on PRODUCT_ASPECT_RE in +// Any mention of the product or of using it, based on the product words in // apps/importer/src/shopee/ai-review-analyzer.ts (isShippingOnlyComment) plus // Shopee's structured product fields. Matched as whole words so "hư" in "như" // or "pin" in "shopping" do not count. Words that also describe delivery @@ -245,7 +245,13 @@ export function evaluateReviewTrustAssessment( | 'verdict' | 'include_in_rating' | 'signals' - > + >, + /** + * The rest of a form v2 review's text, which axis C reads with the + * comment: its verdict line and its writer's pros and cons + * (`rate.review_points`, DOS/Rate#1583). + */ + reviewText?: { verdict?: unknown; points?: readonly unknown[] | null } ): ReviewTrustAssessment { const review = ( rawReview && typeof rawReview === 'object' && !Array.isArray(rawReview) @@ -310,6 +316,8 @@ export function evaluateReviewTrustAssessment( // 3. Axis C: Depth, shared with the review value score (DOS/Rate#1581). const depth: ReviewDepthAxis = reviewDepth({ comment, + verdict: reviewText?.verdict, + points: reviewText?.points ?? undefined, reviewerType: review.reviewer_type, flagCategory: review.flag_category, }); From a4a16dad2651c80934aa4a7828645c0e71cfcffe Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:27:37 +0700 Subject: [PATCH 4/5] feat(shared-data): battery measurements count as technical review depth (#1590) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(web,importer): v2 pros and cons and the Shopee analyzer use reviewDepth (#1583) - The web review reads embed review_points, and the trust weight passes the v2 verdict line and the writer's pros and cons to reviewDepth with the comment; AI-extracted lines stay out. - The Shopee analyzer's axis C is reviewDepth (coin spam as filler), so the web, the analyzer and the review value job classify depth alike. Closes #1583 * test(web): v2 trust depth tests in their own file, with points-only and dedupe cases (#1583 review) The added block pushed core-rate-architecture past tsc's control-flow limit. The new file checks writer lines alone and that a verdict said twice weighs the same; a stale comment on the importer's product regex is fixed. Refs #1583 * feat(shared-data): battery measurements count as technical review depth (#1583) - REVIEW_BATTERY_MEASURE_RE: screen-on time or use in hours tied to the battery or screen, a timed charge between two percentages, and "đo/test thực tế" make a review expert depth, on the web, in the analyzer and in the value score. Hours alone ("giao hàng 2 tiếng") do not, and padding still adds nothing. - The POST response for a v2 review carries a trust assessment computed from its verdict and pros and cons, so the just-submitted card matches the list. Refs #1583 * test(web): the v2 POST card carries a trust assessment from its whole text (#1583) Refs #1583 * Merge remote-tracking branch 'origin/dev' into feat/1583-battery-measure * fix(shared-data): tighter battery measure, at specific_short length (#1583 review) - Gaps stay inside one clause and stop at delivery or waiting words; a spare battery, a torch, a watch or coin cell is not one; "đo/test thực tế" needs a number and "sot" an hours unit. - More real reports match: ranges (6-7 tiếng), charge left after hours, hyphenated screen-on, "sạc 0-100% 50p". - A number never starts inside another, so long digit runs stay linear. - A battery report is expert depth only at specific_short length or more. - The v2 POST card passes lines in their stored form, and its test proves the pros reach the classifier. Refs #1583 Co-Authored-By: Claude Opus 5.5 --- .../api/v1/rate/__tests__/write-paths.test.ts | 8 ++- apps/web/src/app/api/v1/rate/reviews/route.ts | 38 +++++++++---- .../src/__tests__/shared-data.test.ts | 53 +++++++++++++++++++ .../shared-data/src/rules/review-depth.ts | 44 +++++++++++++++ 4 files changed, 133 insertions(+), 10 deletions(-) diff --git a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts index 972362dc..df11be93 100644 --- a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts +++ b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts @@ -394,7 +394,7 @@ async function testReviewsV2() { value: 3.5, }, verdict: 'Hợp người chụp ảnh nhiều, không hợp người chơi game nặng.', - pros: ['Pin dùng trọn ngày'], + pros: ['Pin dùng được 8 tiếng mỗi ngày'], cons: ['Giá hơi cao so với cấu hình'], }; rpcResponder = () => ({ status: 200, json: { id: 'stored' } }); @@ -561,6 +561,12 @@ async function testReviewsV2() { stored.review.signals.criteria_count, Object.keys(v2Review.criteriaRatings).length ); + // The new card's trust reads the verdict and the pros and cons, not the + // empty comment alone (DOS/Rate#1583). + assert.ok(stored.review.trust_assessment, 'the card carries its trust'); + // Only the pro measures the battery, so expert depth proves the lines + // reached the classifier. + assert.equal(stored.review.trust_assessment.depth, 'expert_depth'); const [v2Rpc] = rpcCalls('submit_community_review_v2'); assert.ok(v2Rpc, 'the v2 RPC must be called'); // v1 is never called on the v2 path. diff --git a/apps/web/src/app/api/v1/rate/reviews/route.ts b/apps/web/src/app/api/v1/rate/reviews/route.ts index 3eab0e3b..c0437d6d 100644 --- a/apps/web/src/app/api/v1/rate/reviews/route.ts +++ b/apps/web/src/app/api/v1/rate/reviews/route.ts @@ -34,7 +34,8 @@ import { reviewV2FieldFromRpcDetail, submitCommunityReviewV2, } from '@/server/rate/review-v2'; -import { hasRatingCriteria } from '@repo/shared-data/rules'; +import { hasRatingCriteria, storedPoint } from '@repo/shared-data/rules'; +import { evaluateReviewTrustAssessment } from '@/lib/rate-domain'; import type { AuthenticatedRateRequest } from '@/server/rate/request-auth'; const log = createLogger('api.reviews'); @@ -715,18 +716,37 @@ async function submitReviewV2( {} ); const criteria = v2StoredCriteria(record.criteria_ratings); + const reviewWithCriteria: ReviewItem = { + ...review, + criteria_ratings: criteria, + signals: { + ...review.signals, + criteria_count: Object.keys(criteria).length, + }, + }; + // The new card's trust reads the whole v2 text, as the review list will + // (DOS/Rate#1583); the client would otherwise judge the comment alone. + const submission = checked.submission; + reviewWithCriteria.trust_assessment = evaluateReviewTrustAssessment( + reviewWithCriteria, + { + verdict: record.verdict ?? submission.verdict, + // As rate.review_text_stored keeps them, so the list reads the same. + points: [ + ...submission.pros, + ...submission.cons, + ...Object.values(submission.criterionPoints).flatMap((lines) => [ + ...(lines?.pros ?? []), + ...(lines?.cons ?? []), + ]), + ].map(storedPoint), + } + ); return NextResponse.json( { ok: true, mode: 'supabase_rpc', - review: { - ...review, - criteria_ratings: criteria, - signals: { - ...review.signals, - criteria_count: Object.keys(criteria).length, - }, - }, + review: reviewWithCriteria, }, { headers: REVIEWS_CORS_HEADERS } ); diff --git a/packages/shared-data/src/__tests__/shared-data.test.ts b/packages/shared-data/src/__tests__/shared-data.test.ts index c4e34935..8ddfadf0 100644 --- a/packages/shared-data/src/__tests__/shared-data.test.ts +++ b/packages/shared-data/src/__tests__/shared-data.test.ts @@ -2084,6 +2084,59 @@ function testReviewDepth(): void { rules.reviewDepthText(v2), 'Máy mượt, đáng tiền trong tầm giá.\nPin dùng cả ngày\nSạc hơi chậm' ); + // A measured battery report is technical depth (DOS/Rate#1583). + for (const line of [ + 'Pin dùng được 7 tiếng liên tục, lướt web và xem video', + 'SOT 8 giờ với 5G bật cả ngày, pin ổn hơn máy cũ', + 'Pin trâu sot 9h, chơi game nhẹ buổi tối vẫn còn', + 'Dùng 6h on screen vẫn còn 20%, rất hài lòng', + 'Sạc từ 20% lên 100% mất 45 phút, sạc khá nhanh', + 'Sạc 0-100% 50p, máy hơi ấm khi sạc nhanh', + 'Test thực tế pin trụ 9 tiếng với độ sáng tự động', + 'Màn hình bật 7 tiếng còn 30%, quá ổn cho nhu cầu', + 'Chơi game 5 tiếng tụt 40% pin, máy không nóng lắm', + 'Dùng liên tục 6-7 tiếng mới phải sạc lại, pin ổn', + 'Screen-on time 9 hours on Wi-Fi, very good battery', + 'Onscreen ~7h với 4G, pin tốt trong tầm giá', + ]) { + assert.equal(depth({ comment: line }), 'expert_depth', line); + } + // Hours that are not a battery report, and accessories, are not one. + for (const line of [ + 'Giao hàng 2 tiếng là tới, đóng gói cẩn thận', + 'Đợi 3 tiếng mới được tư vấn, nhân viên chậm', + 'Pin tốt, giao hàng sau 3 tiếng', + 'pin ổn, chờ 2 tiếng mới có hàng', + 'Mua kèm pin dự phòng 10000, giao 2h chiều', + 'Pin con thỏ mua 2 hộp giao 3h', + 'Pin chưa dùng thử, mới mở hộp 1h', + 'Đèn pin sáng 5h liền, cầm chắc tay', + 'Shop test thực tế trước khi giao, rất nhiệt tình', + 'Đo thực tế vừa vặn, áo mặc vừa người', + 'sot 1 sao vì giao chậm quá', + ]) { + assert.notEqual(depth({ comment: line }), 'expert_depth', line); + } + // A measured line counts in a v2 review's pros too. + assert.equal( + depth({ + verdict: 'Máy ổn trong tầm giá, hợp nhu cầu', + points: ['SOT khoảng 7 tiếng mỗi ngày'], + }), + 'expert_depth' + ); + // A bare measurement shorter than specific_short is not expert depth. + assert.notEqual(depth({ comment: 'Pin 7 tiếng' }), 'expert_depth'); + // Long digit runs are scanned once (no quadratic backtracking). + const started = Date.now(); + depth({ comment: `sạc 1% - 1% ${'1'.repeat(100_000)}` }); + depth({ comment: '1'.repeat(100_000) }); + assert.ok(Date.now() - started < 2_000, 'digit runs stay linear'); + // Padding still adds nothing: "pin ok" repeated is no measurement. + assert.equal( + depth({ comment: Array(40).fill('pin ok').join(' ') }), + 'low_effort' + ); // A number ending one part and a unit starting the next are no measurement. assert.equal(depth({ comment: 'Đặt 2', verdict: 'gọn nhẹ' }), 'low_effort'); // Lines extracted by AI from a marketplace comment add nothing. diff --git a/packages/shared-data/src/rules/review-depth.ts b/packages/shared-data/src/rules/review-depth.ts index 86bb5202..d43e1bea 100644 --- a/packages/shared-data/src/rules/review-depth.ts +++ b/packages/shared-data/src/rules/review-depth.ts @@ -33,6 +33,48 @@ export const REVIEW_PRODUCT_ASPECT_RE = export const REVIEW_TECH_MEASURE_RE = /(?:\b\d+\s*(?:mah|wh|w|hz|nit|nits|fps|gb|tb|mp|mm|gram|g|db|ms|°c|độ\s*c)\b|geekbench|antutu|3dmark|cinebench|pcmag|gsmarena|phonearena)/i; +// Pieces of REVIEW_BATTERY_MEASURE_RE. A number never starts inside another +// number, so a long run of digits is scanned once. Every gap stays inside one +// clause: it stops at punctuation and at a delivery or waiting word. +const NOT_WORD_BEFORE = String.raw`(? REVIEW_TECH_MEASURE_RE.test(part)) || + (length >= REVIEW_DEPTH_LENGTH.specificShort && + parts.some((part) => REVIEW_BATTERY_MEASURE_RE.test(part))) || (length >= REVIEW_DEPTH_LENGTH.expert && hasProduct) ) { return 'expert_depth'; From b1dfdabe99873633055a5637e1f87bd343cf4923 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Tue, 3 Nov 2026 03:45:45 +0700 Subject: [PATCH 5/5] feat(web): review quality tier badge and the writer's own value score (#1592) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(web): review quality tier badge and the writer's own value score Review cards show the public tier from rate.reviews.value_tier: an icon and one word for complete and excellent, nothing for basic or unscored. The quality sort puts the tier first, then the existing helpful order. A signed-in writer sees the score and gaps of their own reviews. The page calls the new POST /api/v1/rate/reviews/my-values, which runs rate.my_review_values with the user's JWT. Visitors without a session cookie make no call and do not load the Supabase client. Refs #1547 * fix(web): review value follow-ups from review - A review the rating leaves out shows no tier and does not lead the sort, and a review buyers voted down (more down than up, 3+ votes) does not lead either; the order moves to compareByQuality with tests. - The writer's hint drops stale values on sign-out or failure, hides a gated review's 0, and shows at most three gaps they can act on. - Badge colours no longer match the flag pills; wording follows the design ("bậc", "chỉ bạn thấy"). - OpenAPI documents value_tier on review items; the test script lists the new test once. Refs #1547 Co-Authored-By: Claude Opus 5.5 --- apps/web/locales/cs.json | 12 ++ apps/web/locales/en.json | 12 ++ apps/web/locales/vi.json | 12 ++ apps/web/package.json | 2 +- .../api/v1/rate/__tests__/write-paths.test.ts | 92 +++++++++ .../api/v1/rate/reviews/my-values/route.ts | 93 +++++++++ .../components/reviews/JReviewsSection.tsx | 137 +++++++++++-- .../__tests__/review-value-display.test.ts | 147 ++++++++++++++ apps/web/src/lib/rate-api.ts | 6 + apps/web/src/lib/rate-domain-core.ts | 3 + apps/web/src/lib/review-value-display.ts | 192 ++++++++++++++++++ apps/web/src/server/rate/openapi.ts | 43 ++++ 12 files changed, 734 insertions(+), 17 deletions(-) create mode 100644 apps/web/src/app/api/v1/rate/reviews/my-values/route.ts create mode 100644 apps/web/src/lib/__tests__/review-value-display.test.ts create mode 100644 apps/web/src/lib/review-value-display.ts diff --git a/apps/web/locales/cs.json b/apps/web/locales/cs.json index ebe4347b..14ef6d74 100644 --- a/apps/web/locales/cs.json +++ b/apps/web/locales/cs.json @@ -546,6 +546,18 @@ "noMatchingReviews": "Žádné recenze neodpovídají zvolenému filtru.", "roleExpertBadge": "Expert Rate", "roleBuyerBadge": "Kupující", + "valueTierTitle": "Úroveň kvality recenze", + "valueTierComplete": "Úplná", + "valueTierExcellent": "Výborná", + "myReviewValue": "Hodnota vaší recenze (vidíte jen vy): {score}/100", + "myReviewValueGaps": "Doplňte: {gaps}", + "valueGapDetail": "podrobnosti", + "valueGapCriteria": "všechna kritéria", + "valueGapVerdict": "závěr", + "valueGapProsCons": "klady a zápory", + "valueGapCriterionPoints": "klady a zápory u kritérií", + "valueGapPhoto": "skutečnou fotku", + "valueGapVerified": "ověřený nákup", "routedToMerchantBadge": "Přesměrováno do skóre prodejce", "sponsorshipBadgeSponsoredBalanced": "Sponzorováno", "sponsorshipBadgeSponsoredPromo": "Reklama", diff --git a/apps/web/locales/en.json b/apps/web/locales/en.json index 887856d5..18332415 100644 --- a/apps/web/locales/en.json +++ b/apps/web/locales/en.json @@ -546,6 +546,18 @@ "noMatchingReviews": "No reviews match the current filter criteria.", "roleExpertBadge": "Rate expert", "roleBuyerBadge": "Buyer", + "valueTierTitle": "Review quality tier", + "valueTierComplete": "Complete", + "valueTierExcellent": "Excellent", + "myReviewValue": "Your review value (only you see it): {score}/100", + "myReviewValueGaps": "Add: {gaps}", + "valueGapDetail": "detail", + "valueGapCriteria": "all criteria", + "valueGapVerdict": "a verdict", + "valueGapProsCons": "pros and cons", + "valueGapCriterionPoints": "pros and cons per criterion", + "valueGapPhoto": "a real photo", + "valueGapVerified": "a verified purchase", "routedToMerchantBadge": "Routed to merchant score", "sponsorshipBadgeSponsoredBalanced": "Sponsored", "sponsorshipBadgeSponsoredPromo": "Advertising", diff --git a/apps/web/locales/vi.json b/apps/web/locales/vi.json index 77185f1b..7d35d1d3 100644 --- a/apps/web/locales/vi.json +++ b/apps/web/locales/vi.json @@ -546,6 +546,18 @@ "noMatchingReviews": "Không có đánh giá nào khớp với bộ lọc hiện tại.", "roleExpertBadge": "Chuyên gia Rate", "roleBuyerBadge": "Người mua", + "valueTierTitle": "Bậc chất lượng của đánh giá", + "valueTierComplete": "Đầy đủ", + "valueTierExcellent": "Xuất sắc", + "myReviewValue": "Điểm giá trị bài của bạn (chỉ bạn thấy): {score}/100", + "myReviewValueGaps": "Thêm: {gaps}", + "valueGapDetail": "chi tiết", + "valueGapCriteria": "chấm đủ tiêu chí", + "valueGapVerdict": "kết luận", + "valueGapProsCons": "cả ưu lẫn nhược", + "valueGapCriterionPoints": "ưu nhược theo tiêu chí", + "valueGapPhoto": "ảnh thật", + "valueGapVerified": "xác nhận đã mua", "routedToMerchantBadge": "Chuyển sang điểm người bán", "sponsorshipBadgeSponsoredBalanced": "Có tài trợ", "sponsorshipBadgeSponsoredPromo": "Quảng cáo", diff --git a/apps/web/package.json b/apps/web/package.json index ced10a3b..c5ce3b27 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -8,7 +8,7 @@ "lint": "eslint . --max-warnings 0", "preview": "next build && next start", "start": "next start", - "test": "pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/core-rate-architecture.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-model.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/write-paths.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/spec-scope.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/item-fields.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/logger.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-form-v2.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/concurrency.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-profile-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-review-risk.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/watchlist-sizes.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-compare.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-variant-links.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/compare-params.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/catalog-listing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/discount-verdict-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-detail-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-competitors.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/analyzer-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sold-count.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/content-lang.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/image-url.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/client-messages.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/seo.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/offer-plausibility.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-market-stats.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sitemap-lastmod.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-badge-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-criteria.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/trust-depth-v2.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/rating-score.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/rating-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/focus-trap.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/display-truth.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/spec-template.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-spec-rows.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sale-state.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/csp.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/device-images.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/render-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/model-match.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/developers-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/plural-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/locale-format.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/paged-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/server-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/hero-products.test.ts", + "test": "pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/core-rate-architecture.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-model.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/write-paths.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/spec-scope.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/item-fields.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/logger.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-form-v2.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-value-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/concurrency.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-profile-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-review-risk.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/watchlist-sizes.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-compare.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-variant-links.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/compare-params.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/catalog-listing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/discount-verdict-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-detail-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-competitors.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/analyzer-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sold-count.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/content-lang.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/image-url.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/client-messages.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/seo.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/offer-plausibility.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-market-stats.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sitemap-lastmod.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-badge-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-criteria.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/trust-depth-v2.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/rating-score.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/rating-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/focus-trap.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/display-truth.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/spec-template.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-spec-rows.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sale-state.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/csp.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/device-images.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/render-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/model-match.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/developers-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/plural-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/locale-format.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/paged-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/server-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/hero-products.test.ts", "typecheck": "tsc --noEmit" }, "browserslist": [ diff --git a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts index df11be93..8524c91d 100644 --- a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts +++ b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts @@ -738,6 +738,97 @@ async function testReviewsByIdPublishedOnly() { ); } +async function testMyReviewValues() { + const { POST } = await import('../reviews/my-values/route'); + const MINE = 'aaaaaaaa-1111-4222-8333-444444444444'; + const OTHER = 'aaaaaaaa-1111-4222-8333-555555555555'; + const breakdown = { + version: 1, + gated: false, + score: 62, + tier: 'complete', + gaps: ['photo', 'not_a_gap', 'verified', 'photo'], + }; + + // Nobody signed in: no call to the database. + calls = []; + let res = await POST(post('/api/v1/rate/reviews/my-values', { ids: [MINE] })); + assert.equal(res.status, 401); + assert.equal(rpcCalls('my_review_values').length, 0); + + // Bad ids are refused before the database. + for (const ids of [ + 'x', + [MINE, 'not-a-uuid'], + Array.from( + { length: 101 }, + (_, i) => `aaaaaaaa-1111-4222-8333-${String(i).padStart(12, '0')}` + ), + ]) { + calls = []; + res = await POST( + post('/api/v1/rate/reviews/my-values', { ids }, GOOD_TOKEN) + ); + assert.equal(res.status, 400); + assert.equal(rpcCalls('my_review_values').length, 0); + } + + // The writer's call goes out with their own token; the answer keeps only + // asked ids with a valid score, the known gaps, and is never cached. + calls = []; + rpcResponder = (name) => + name === 'my_review_values' + ? { + status: 200, + json: [ + { review_id: MINE, value_score: 61.6, value_breakdown: breakdown }, + { + review_id: 'aaaaaaaa-1111-4222-8333-666666666666', + value_score: 90, + value_breakdown: breakdown, + }, + { review_id: OTHER, value_score: 'NaN', value_breakdown: null }, + ], + } + : { status: 500, json: { message: 'unexpected rpc' } }; + res = await POST( + post( + '/api/v1/rate/reviews/my-values', + { ids: [MINE, MINE.toUpperCase(), OTHER] }, + GOOD_TOKEN + ) + ); + assert.equal(res.status, 200); + assert.match(res.headers.get('cache-control') ?? '', /no-store/); + assert.deepEqual(await res.json(), { + ok: true, + values: [{ id: MINE, score: 62, gaps: ['photo', 'verified'] }], + }); + const [call] = rpcCalls('my_review_values'); + assert.ok(call, 'my_review_values must be called'); + assert.equal(call.headers.authorization, `Bearer ${GOOD_TOKEN}`); + assert.deepEqual(call.body, { p_review_ids: [MINE, OTHER] }); + assert.ok( + calls.every((c) => !JSON.stringify(c.headers).includes(SERVICE_KEY)), + 'no request may carry the service-role key' + ); + + // A missing function and other failures are reported, not faked. + rpcResponder = () => ({ + status: 404, + json: { code: 'PGRST202', message: 'Could not find the function' }, + }); + res = await POST( + post('/api/v1/rate/reviews/my-values', { ids: [MINE] }, GOOD_TOKEN) + ); + assert.equal(res.status, 503); + rpcResponder = () => ({ status: 500, json: { code: 'XX000' } }); + res = await POST( + post('/api/v1/rate/reviews/my-values', { ids: [MINE] }, GOOD_TOKEN) + ); + assert.equal(res.status, 502); +} + async function testWatchlist() { const { POST } = await import('../watchlist/route'); type Req = Parameters[0]; @@ -2028,6 +2119,7 @@ async function main() { await testReviews(); await testReviewsV2(); await testReviewsByIdPublishedOnly(); + await testMyReviewValues(); await testWatchlist(); await testCookieSessions(); await testFilterBoundsAndWatchlistPrices(); diff --git a/apps/web/src/app/api/v1/rate/reviews/my-values/route.ts b/apps/web/src/app/api/v1/rate/reviews/my-values/route.ts new file mode 100644 index 00000000..e426fc46 --- /dev/null +++ b/apps/web/src/app/api/v1/rate/reviews/my-values/route.ts @@ -0,0 +1,93 @@ +import { NextResponse } from 'next/server'; +import { + PUBLIC_API_CORS_HEADERS, + payloadTooLargeResponse, + readBoundedJsonBody, +} from '@/server/rate/api-handler'; +import { + authenticateRateRequest, + isMissingRpcError, +} from '@/server/rate/request-auth'; +import { createLogger, serializeError } from '@/lib/logger'; +import { parseMyReviewValue, parseReviewIds } from '@/lib/review-value-display'; + +const log = createLogger('api.reviews.my-values'); + +// The answer belongs to one signed-in writer: never cached, never shared. +const MY_VALUES_HEADERS: Record = { + ...PUBLIC_API_CORS_HEADERS, + 'Access-Control-Allow-Methods': 'POST, OPTIONS', + 'Cache-Control': 'private, no-store', +}; + +export async function OPTIONS() { + return new NextResponse(null, { status: 204, headers: MY_VALUES_HEADERS }); +} + +function fail(status: number, code: string, error: string) { + return NextResponse.json( + { ok: false, error, code }, + { status, headers: MY_VALUES_HEADERS } + ); +} + +/** + * POST { ids: uuid[] }: the value score and gaps of the caller's own reviews + * among `ids` (DOS/Rate#1547, D7). It calls rate.my_review_values with the + * caller's JWT, so the database answers only for reviews they wrote; other + * ids are left out of the answer. + */ +export async function POST(request: Request) { + const auth = await authenticateRateRequest( + request, + 'read your review values', + MY_VALUES_HEADERS + ); + if (!auth.ok) return auth.response; + + const body = await readBoundedJsonBody(request, 8 * 1024); + if (body.tooLarge) return payloadTooLargeResponse(MY_VALUES_HEADERS); + const ids = parseReviewIds( + body.value && typeof body.value === 'object' && !Array.isArray(body.value) + ? (body.value as { ids?: unknown }).ids + : undefined + ); + if (!ids) { + return fail(400, 'INVALID_IDS', 'Provide ids: up to 100 review uuids'); + } + if (ids.length === 0) { + return NextResponse.json( + { ok: true, values: [] }, + { headers: MY_VALUES_HEADERS } + ); + } + + const { data, error } = await auth.rateClient.rpc('my_review_values', { + p_review_ids: ids, + }); + if (error) { + if (isMissingRpcError(error)) { + return fail( + 503, + 'REVIEW_VALUES_UNAVAILABLE', + 'Review values unavailable' + ); + } + log.error('my_review_values failed', { error: serializeError(error) }); + return fail(502, 'READ_FAILED', 'Could not read review values'); + } + + const asked = new Set(ids); + const values = []; + for (const row of Array.isArray(data) ? data : []) { + const id = + typeof row?.review_id === 'string' ? row.review_id.toLowerCase() : ''; + if (!asked.has(id)) continue; + const value = parseMyReviewValue(row.value_score, row.value_breakdown); + if (value) values.push({ id, ...value }); + } + return NextResponse.json( + { ok: true, values }, + { headers: MY_VALUES_HEADERS } + ); +} diff --git a/apps/web/src/components/reviews/JReviewsSection.tsx b/apps/web/src/components/reviews/JReviewsSection.tsx index dc59f44a..8bdb5864 100644 --- a/apps/web/src/components/reviews/JReviewsSection.tsx +++ b/apps/web/src/components/reviews/JReviewsSection.tsx @@ -6,6 +6,7 @@ import ExportMenu from '@/components/elementary/ExportMenu'; import PhoneCollapsible from '@/components/product/PhoneCollapsible'; import StarRating from '@/components/elementary/StarRating'; import { + BsAward, BsBarChart, BsCheckCircle, BsExclamationTriangle, @@ -14,6 +15,7 @@ import { BsHandThumbsUp, BsInfoCircle, BsMegaphone, + BsPatchCheck, BsRobot, BsStarFill, } from 'react-icons/bs'; @@ -55,6 +57,18 @@ import AnalyzerText, { analyzerText } from './AnalyzerText'; import ReviewFormV2 from './ReviewFormV2'; import { isReviewFormV2Enabled } from '@/lib/review-form-v2'; import { hasRatingCriteria } from '@repo/shared-data/rules'; +import { + hasSessionCookie, + isReviewId, + MY_REVIEW_VALUES_MAX_IDS, + parseMyReviewValue, + compareByQuality, + reviewTierBadgeKey, + reviewValueGapKey, + shownReviewTier, + shownReviewValueGaps, + type MyReviewValue, +} from '@/lib/review-value-display'; import { formatDecimal, formatPercent, @@ -228,6 +242,8 @@ export default function JReviewsSection({ const [authNotice, setAuthNotice] = useState(null); const [activeTab, setActiveTab] = useState('all'); const [sortMode, setSortMode] = useState('weight_desc'); + // The signed-in writer's own value scores, by review id (DOS/Rate#1547). + const [myValues, setMyValues] = useState>({}); const [searchKeyword, setSearchKeyword] = useState(''); const [starFilter, setStarFilter] = useState<'all' | 5 | 4 | 3>('all'); @@ -434,22 +450,8 @@ export default function JReviewsSection({ return [...filtered].sort((a, b) => { if (sortMode === 'weight_desc') { - const wilsonA = computeWilsonLowerBound(a.upvotes, a.downvotes); - const wilsonB = computeWilsonLowerBound(b.upvotes, b.downvotes); - if ( - a.include_in_rating && - b.include_in_rating && - Math.abs(wilsonB - wilsonA) >= 0.005 - ) { - return wilsonB - wilsonA; - } - if (b.helpfulness_weight !== a.helpfulness_weight) { - return b.helpfulness_weight - a.helpfulness_weight; - } - return ( - new Date(b.created_at || 0).getTime() - - new Date(a.created_at || 0).getTime() - ); + // The public quality tier leads (DOS/Rate#1547, D7). + return compareByQuality(a, b, computeWilsonLowerBound); } if (sortMode === 'rating_desc') { return b.rating_star - a.rating_star; @@ -464,6 +466,59 @@ export default function JReviewsSection({ }); }, [reviews, activeTab, sortMode, searchKeyword, starFilter, tAnalyzer]); + // Ids of the reviews on the page, stable across votes, for the lookup below. + const reviewIdsKey = useMemo( + () => + [...new Set(reviews.map((r) => r.id).filter(isReviewId))] + .slice(0, MY_REVIEW_VALUES_MAX_IDS) + .sort() + .join(','), + [reviews] + ); + + // A signed-in writer sees the value score and gaps of their own reviews + // (DOS/Rate#1547, D7). Visitors without a session cookie make no call and + // never load the Supabase client. + useEffect(() => { + let cancelled = false; + void (async () => { + // Without a session, or on any failure, no number stays on screen. + let next: Record = {}; + try { + const token = + reviewIdsKey && hasSessionCookie(document.cookie) + ? await getBrowserAccessToken() + : null; + if (token && !cancelled) { + const res = await fetch('/api/v1/rate/reviews/my-values', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}`, + }, + body: JSON.stringify({ ids: reviewIdsKey.split(',') }), + }); + const payload = res.ok ? await res.json() : null; + for (const row of Array.isArray(payload?.values) + ? payload.values + : []) { + const value = parseMyReviewValue(row?.score, { gaps: row?.gaps }); + if (isReviewId(row?.id) && value) { + next[row.id.toLowerCase()] = value; + } + } + } + } catch { + // The hint is optional; the page works without it. + next = {}; + } + if (!cancelled) setMyValues(next); + })(); + return () => { + cancelled = true; + }; + }, [reviewIdsKey]); + async function handleVote(reviewId: string, clickedVote: VoteChoice) { if (pendingIds[reviewId]) return; @@ -1586,6 +1641,15 @@ export default function JReviewsSection({ const isPending = Boolean(pendingIds[rev.id]); const isZeroWeight = !rev.include_in_rating || rev.helpfulness_weight <= 0; + const tier = shownReviewTier(rev); + const tierKey = reviewTierBadgeKey(tier); + const myValue = myValues[rev.id.toLowerCase()]; + const myGaps = myValue + ? shownReviewValueGaps( + myValue.gaps, + hasRatingCriteria(reviewCategorySlug ?? categorySlug) + ) + : []; return (
@@ -1663,8 +1727,49 @@ export default function JReviewsSection({ ), })} + {tierKey && ( + + {tier === 'excellent' ? ( + + )}
+ {/* The writer's own number and what would raise it; the + public sees only the tier (DOS/Rate#1547, D7). */} + {/* A gated review scores 0 with no gaps: its flag says why. */} + {myValue && (myValue.score > 0 || myGaps.length > 0) && ( +
+ {t('myReviewValue', { score: myValue.score })} + {myGaps.length > 0 && + ` · ${t('myReviewValueGaps', { + gaps: myGaps + .map((gap) => reviewValueGapKey(gap)) + .filter((key) => key !== null) + .map((key) => t(key)) + .join(', '), + })}`} +
+ )} + {rev.variation && (
{t.rich('variationPrefix', { diff --git a/apps/web/src/lib/__tests__/review-value-display.test.ts b/apps/web/src/lib/__tests__/review-value-display.test.ts new file mode 100644 index 00000000..695160e7 --- /dev/null +++ b/apps/web/src/lib/__tests__/review-value-display.test.ts @@ -0,0 +1,147 @@ +import assert from 'node:assert/strict'; +import { + compareByQuality, + hasSessionCookie, + isReviewId, + parseMyReviewValue, + parseReviewIds, + parseReviewValueTier, + reviewTierBadgeKey, + reviewTierRank, + reviewValueGapKey, + shownReviewTier, + shownReviewValueGaps, +} from '../review-value-display'; + +const ID = 'aaaaaaaa-1111-4222-8333-444444444444'; + +// The public tier: only the three stored values. +assert.equal(parseReviewValueTier('excellent'), 'excellent'); +assert.equal(parseReviewValueTier('complete'), 'complete'); +assert.equal(parseReviewValueTier('basic'), 'basic'); +for (const raw of ['Excellent', '', null, undefined, 3, '__proto__']) { + assert.equal(parseReviewValueTier(raw), null); +} + +// Excellent before complete; basic and unscored rank together, last. +assert.ok(reviewTierRank('excellent') > reviewTierRank('complete')); +assert.ok(reviewTierRank('complete') > reviewTierRank('basic')); +assert.equal(reviewTierRank(null), reviewTierRank('basic')); +const sorted = [null, 'excellent', 'basic', 'complete'] + .map((tier) => parseReviewValueTier(tier)) + .sort((a, b) => reviewTierRank(b) - reviewTierRank(a)); +assert.deepEqual(sorted.slice(0, 2), ['excellent', 'complete']); + +// A review the rating leaves out keeps no tier, so it cannot be lifted. +const kept = { value_tier: 'excellent' as const, include_in_rating: true }; +assert.equal(shownReviewTier({ ...kept, helpfulness_weight: 1 }), 'excellent'); +assert.equal(shownReviewTier({ ...kept, helpfulness_weight: 0 }), null); +assert.equal( + shownReviewTier({ ...kept, include_in_rating: false, helpfulness_weight: 1 }), + null +); +assert.equal( + shownReviewTier({ include_in_rating: true, helpfulness_weight: 1 }), + null +); + +// The "most helpful" order: tier first, unless buyers voted it down; then +// votes, weight and the newest. +const wilson = (up: number, down: number) => + up + down === 0 ? 0 : up / (up + down + 2); +const review = ( + id: string, + extra: Partial<{ + value_tier: 'basic' | 'complete' | 'excellent' | null; + include_in_rating: boolean; + helpfulness_weight: number; + upvotes: number; + downvotes: number; + created_at: string; + }> = {} +) => ({ + id, + value_tier: null, + include_in_rating: true, + helpfulness_weight: 1, + upvotes: 0, + downvotes: 0, + created_at: '2026-11-01T00:00:00Z', + ...extra, +}); +const order = (list: ReturnType[]) => + [...list].sort((a, b) => compareByQuality(a, b, wilson)).map((r) => r.id); +assert.deepEqual( + order([ + review('marketplace', { upvotes: 300 }), + review('complete', { value_tier: 'complete' }), + review('excellent', { value_tier: 'excellent' }), + review('voted-down', { value_tier: 'excellent', downvotes: 40 }), + review('flagged', { value_tier: 'excellent', include_in_rating: false }), + ]), + ['excellent', 'complete', 'marketplace', 'voted-down', 'flagged'] +); +assert.deepEqual( + order([ + review('older', { value_tier: 'complete' }), + review('newer', { + value_tier: 'complete', + created_at: '2026-11-02T00:00:00Z', + }), + ]), + ['newer', 'older'] +); + +// The writer sees at most three gaps they can act on. +assert.deepEqual( + shownReviewValueGaps( + ['detail', 'criteria', 'verdict', 'pros_cons', 'photo', 'verified'], + true + ), + ['detail', 'criteria', 'verdict'] +); +assert.deepEqual( + shownReviewValueGaps( + ['criteria', 'criterion_points', 'photo', 'verified'], + false + ), + ['photo'] +); + +// Less text: basic and unscored reviews carry no badge. +assert.equal(reviewTierBadgeKey('excellent'), 'valueTierExcellent'); +assert.equal(reviewTierBadgeKey('complete'), 'valueTierComplete'); +assert.equal(reviewTierBadgeKey('basic'), null); +assert.equal(reviewTierBadgeKey(null), null); + +assert.equal(reviewValueGapKey('pros_cons'), 'valueGapProsCons'); +assert.equal(reviewValueGapKey('toString'), null); +assert.equal(reviewValueGapKey(1), null); + +// Review ids: uuids only, at most 100, deduplicated case-insensitively. +assert.ok(isReviewId(ID)); +assert.ok(!isReviewId('stored')); +assert.deepEqual(parseReviewIds([ID, ID.toUpperCase()]), [ID]); +assert.deepEqual(parseReviewIds([]), []); +assert.equal(parseReviewIds([ID, 'x']), null); +assert.equal(parseReviewIds(ID), null); +assert.equal(parseReviewIds(new Array(101).fill(ID)), null); + +// The writer's value: a 0-100 score, rounded, and the known gaps once each. +assert.deepEqual( + parseMyReviewValue('61.5', { gaps: ['photo', 'x', 'photo', 'verified'] }), + { score: 62, gaps: ['photo', 'verified'] } +); +assert.deepEqual(parseMyReviewValue(0, null), { score: 0, gaps: [] }); +for (const score of [-1, 101, Number.NaN, 'abc', null, undefined]) { + assert.equal(parseMyReviewValue(score, { gaps: [] }), null); +} + +// Only a non-empty Supabase auth cookie counts as a session. +assert.ok(hasSessionCookie('a=1; sb-abc-auth-token=xyz')); +assert.ok(hasSessionCookie('sb-abc-auth-token.0=base64-xyz')); +assert.ok(!hasSessionCookie('sb-abc-auth-token=')); +assert.ok(!hasSessionCookie('x-sb-abc-auth-token=1')); +assert.ok(!hasSessionCookie('')); + +console.log('✅ review value display (tier, ids, writer value) passed'); diff --git a/apps/web/src/lib/rate-api.ts b/apps/web/src/lib/rate-api.ts index 014b80ef..5ad5e157 100644 --- a/apps/web/src/lib/rate-api.ts +++ b/apps/web/src/lib/rate-api.ts @@ -4,6 +4,7 @@ import { createLogger } from '@/lib/logger'; import { timeStep } from '@/lib/server-timing'; import { rankModelCandidates } from '@/lib/model-match'; import { isPendingReview } from '@/lib/review-display'; +import { parseReviewValueTier } from '@/lib/review-value-display'; import { listingReviewSummary } from '@/lib/analyzer-copy'; import { countBrandsByItem } from '@/lib/item-directory'; import { masterSiblingKeyPrefix } from '@/lib/product-variant-links'; @@ -1719,6 +1720,8 @@ interface RawReviewRow { /** The form v2 verdict line; the analyzer's verdict is in review_flags. */ verdict?: string | null; review_points?: RawReviewPointRow[] | null; + /** The public quality tier (DOS/Rate#1547). */ + value_tier?: string | null; } /** @@ -1907,6 +1910,7 @@ export async function getMasterProductReviews( flag_category: flagCategory, reason: vf?.reason || null, signals: vf?.signals || null, + value_tier: parseReviewValueTier(r.value_tier), }; item.trust_assessment = evaluateReviewTrustAssessment(item, { verdict: r.verdict, @@ -3344,6 +3348,7 @@ export async function getMerchantsList( flag_category: flagCategory, reason: vf?.reason || null, signals: vf?.signals || null, + value_tier: parseReviewValueTier(r.value_tier), }; const existing = mappedReviewsByMerchant.get(mId) ?? []; @@ -3907,6 +3912,7 @@ export async function getMerchantProfileBase( flag_category: flagCategory, reason: vf?.reason || null, signals: vf?.signals || null, + value_tier: parseReviewValueTier(r.value_tier), }; item.trust_assessment = evaluateReviewTrustAssessment(item, { verdict: r.verdict, diff --git a/apps/web/src/lib/rate-domain-core.ts b/apps/web/src/lib/rate-domain-core.ts index e97cee70..03d8160b 100644 --- a/apps/web/src/lib/rate-domain-core.ts +++ b/apps/web/src/lib/rate-domain-core.ts @@ -5,6 +5,7 @@ * them (DOS/Rate#756). */ import type { MasterProduct } from '@repo/shared-data'; +import type { ReviewValueTier } from '@repo/shared-data/rules'; import { REVIEW_PLATFORM_LABEL_RE, REVIEW_PRODUCT_ASPECT_RE, @@ -148,6 +149,8 @@ export interface ReviewItem { reason: string | null; signals: Record | null; trust_assessment?: ReviewTrustAssessment; + /** The public quality tier (DOS/Rate#1547, D7); null when unscored. */ + value_tier?: ReviewValueTier | null; } export function toSafeNonNegativeVoteCount( diff --git a/apps/web/src/lib/review-value-display.ts b/apps/web/src/lib/review-value-display.ts new file mode 100644 index 00000000..2879a332 --- /dev/null +++ b/apps/web/src/lib/review-value-display.ts @@ -0,0 +1,192 @@ +import type { ReviewValueGap, ReviewValueTier } from '@repo/shared-data/rules'; + +/** + * Review value on the web (DOS/Rate#1547, D7). The public sees a review's + * tier (`rate.reviews.value_tier`); only the writer sees the number and what + * would raise it, read through `rate.my_review_values` as that writer. + */ + +const TIERS: readonly ReviewValueTier[] = ['basic', 'complete', 'excellent']; + +/** The tier of a stored review, null when it is unscored, gated or unknown. */ +export function parseReviewValueTier(raw: unknown): ReviewValueTier | null { + return typeof raw === 'string' && (TIERS as readonly string[]).includes(raw) + ? (raw as ReviewValueTier) + : null; +} + +/** Sort rank of a tier: an unscored review ranks with a basic one. */ +export function reviewTierRank(tier: ReviewValueTier | null | undefined) { + return tier === 'excellent' ? 2 : tier === 'complete' ? 1 : 0; +} + +/** + * The tier a review shows and sorts by. A review the rating leaves out (a + * flag or zero weight since it was scored) shows none, so a stale tier never + * lifts it. + */ +export function shownReviewTier(review: { + value_tier?: ReviewValueTier | null; + include_in_rating: boolean; + helpfulness_weight: number; +}): ReviewValueTier | null { + return review.include_in_rating && review.helpfulness_weight > 0 + ? (review.value_tier ?? null) + : null; +} + +interface QualitySortedReview { + value_tier?: ReviewValueTier | null; + include_in_rating: boolean; + helpfulness_weight: number; + upvotes: number; + downvotes: number; + created_at: string | null; +} + +/** Buyers voted it down: more down than up votes, at least 3 in all. */ +function votedDown(r: { upvotes: number; downvotes: number }): boolean { + return r.downvotes > r.upvotes && r.upvotes + r.downvotes >= 3; +} + +/** + * The "most helpful" order: the shown tier first (design 6.3), unless buyers + * voted the review down, so votes still outrank a complete form; then the + * helpful-vote order and the newest. `wilson` is the Wilson lower bound of a + * review's votes. + */ +export function compareByQuality( + a: T, + b: T, + wilson: (upvotes: number, downvotes: number) => number +): number { + const rank = (r: T) => + votedDown(r) ? 0 : reviewTierRank(shownReviewTier(r)); + const tierGap = rank(b) - rank(a); + if (tierGap !== 0) return tierGap; + const wilsonA = wilson(a.upvotes, a.downvotes); + const wilsonB = wilson(b.upvotes, b.downvotes); + if ( + a.include_in_rating && + b.include_in_rating && + Math.abs(wilsonB - wilsonA) >= 0.005 + ) { + return wilsonB - wilsonA; + } + if (b.helpfulness_weight !== a.helpfulness_weight) { + return b.helpfulness_weight - a.helpfulness_weight; + } + return ( + new Date(b.created_at || 0).getTime() - + new Date(a.created_at || 0).getTime() + ); +} + +/** + * The gaps the writer is shown: at most three they can act on. A verified + * purchase has no path from the form yet, and criteria gaps need a category + * with a criteria list. + */ +export function shownReviewValueGaps( + gaps: readonly ReviewValueGap[], + hasCriteria: boolean +): ReviewValueGap[] { + return gaps + .filter( + (gap) => + gap !== 'verified' && + (hasCriteria || (gap !== 'criteria' && gap !== 'criterion_points')) + ) + .slice(0, 3); +} + +/** The badge a tier shows: basic and unscored show none (less text). */ +export function reviewTierBadgeKey( + tier: ReviewValueTier | null | undefined +): 'valueTierExcellent' | 'valueTierComplete' | null { + return tier === 'excellent' + ? 'valueTierExcellent' + : tier === 'complete' + ? 'valueTierComplete' + : null; +} + +const GAP_KEYS = { + detail: 'valueGapDetail', + criteria: 'valueGapCriteria', + verdict: 'valueGapVerdict', + pros_cons: 'valueGapProsCons', + criterion_points: 'valueGapCriterionPoints', + photo: 'valueGapPhoto', + verified: 'valueGapVerified', +} as const satisfies Record; + +export type ReviewValueGapKey = (typeof GAP_KEYS)[ReviewValueGap]; + +/** The locale key naming a gap, null for a name this build does not know. */ +export function reviewValueGapKey(gap: unknown): ReviewValueGapKey | null { + return typeof gap === 'string' && Object.hasOwn(GAP_KEYS, gap) + ? GAP_KEYS[gap as ReviewValueGap] + : null; +} + +/** True for a review id the value lookup can carry. */ +export function isReviewId(id: unknown): id is string { + return typeof id === 'string' && UUID_RE.test(id); +} + +/** What the writer of a review sees about its value. */ +export interface MyReviewValue { + score: number; + gaps: ReviewValueGap[]; +} + +/** At most this many ids per call, as `rate.my_review_values` allows. */ +export const MY_REVIEW_VALUES_MAX_IDS = 100; + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** The distinct review ids of a request, null when any is not a uuid. */ +export function parseReviewIds(raw: unknown): string[] | null { + if (!Array.isArray(raw) || raw.length > MY_REVIEW_VALUES_MAX_IDS) { + return null; + } + const ids = new Set(); + for (const id of raw) { + if (!isReviewId(id)) return null; + ids.add(id.toLowerCase()); + } + return [...ids]; +} + +/** + * One row of `rate.my_review_values` (or of the route's answer) as the + * writer's value; null for a row that is not a scored review. + */ +export function parseMyReviewValue( + score: unknown, + breakdown: unknown +): MyReviewValue | null { + const n = typeof score === 'string' ? Number(score) : score; + if (typeof n !== 'number' || !Number.isFinite(n) || n < 0 || n > 100) { + return null; + } + const rawGaps = + breakdown && typeof breakdown === 'object' && !Array.isArray(breakdown) + ? (breakdown as { gaps?: unknown }).gaps + : undefined; + const gaps = Array.isArray(rawGaps) + ? (rawGaps.filter((g) => reviewValueGapKey(g) !== null) as ReviewValueGap[]) + : []; + return { score: Math.round(n), gaps: [...new Set(gaps)] }; +} + +/** True when the browser holds a Supabase session cookie. */ +export function hasSessionCookie(cookie: string): boolean { + return typeof cookie === 'string' + ? /(?:^|;\s*)sb-[a-z0-9_-]{1,64}-auth-token(?:\.\d{1,3})?=[^;\s]/i.test( + cookie + ) + : false; +} diff --git a/apps/web/src/server/rate/openapi.ts b/apps/web/src/server/rate/openapi.ts index 6d7235c2..a7971fcc 100644 --- a/apps/web/src/server/rate/openapi.ts +++ b/apps/web/src/server/rate/openapi.ts @@ -1693,6 +1693,43 @@ function buildBaseRateOpenApiSpec( }, }, }, + '/api/v1/rate/reviews/my-values': { + post: { + operationId: 'getMyReviewValues', + summary: "Read the value score of the caller's own reviews", + description: + "Returns the 0-100 value score and the missing parts (`gaps`) of the reviews among `ids` that the signed-in caller wrote, through `rate.my_review_values` with the caller's JWT. Other ids are left out. The public sees only each review's `value_tier`. Never cached.", + tags: ['Community Reviews & Voting'], + security: [{ bearerAuth: [] }], + requestBody: { + required: true, + content: { + 'application/json': { + schema: { + type: 'object', + required: ['ids'], + properties: { + ids: { + type: 'array', + maxItems: 100, + items: { type: 'string', format: 'uuid' }, + }, + }, + }, + }, + }, + }, + responses: { + '200': { + description: + "`values`: `{ id, score, gaps }` for each of the caller's own reviews that has a score.", + }, + '400': errorResponse('`ids` is not a list of up to 100 uuids.'), + '401': errorResponse('Missing or invalid bearer token.'), + '503': errorResponse('Review values are not available here.'), + }, + }, + }, '/api/v1/rate/stats': { get: { operationId: 'getPlatformStats', @@ -2359,6 +2396,12 @@ function buildBaseRateOpenApiSpec( ], }, reason: { type: ['string', 'null'] }, + value_tier: { + type: ['string', 'null'], + enum: ['basic', 'complete', 'excellent', null], + description: + 'Public quality tier of the review; null when it is not scored yet or is gated. The 0-100 value score is private to the writer.', + }, }, }, },