diff --git a/apps/web/src/server/rate/openapi.ts b/apps/web/src/server/rate/openapi.ts index a7971fcc..5d04382a 100644 --- a/apps/web/src/server/rate/openapi.ts +++ b/apps/web/src/server/rate/openapi.ts @@ -430,7 +430,7 @@ function buildBaseRateOpenApiSpec( name: 'minRating', in: 'query', description: - 'Minimum effective product rating (0-5 star scale); alias `rating` is also accepted', + 'Minimum product rating (0-5 star scale), compared with `rating_score_5`, the rating every Rate page shows, at one decimal: a product shown as 4.5 passes 4.5. A product without a rating never passes. Alias `rating` is also accepted', schema: { type: 'number' }, }, { @@ -450,7 +450,8 @@ function buildBaseRateOpenApiSpec( { name: 'sort', in: 'query', - description: 'Sort order for returned master products', + description: + 'Sort order for returned master products. `rating_desc` orders by the rating every Rate page shows (`rating_score_5` before its rounding to one decimal), products without a rating last', schema: { type: 'string', enum: [ diff --git a/supabase/migrations/20261104120000_rate_search_master_products_trgm_index_and_numeric_guard.sql b/supabase/migrations/20261104120000_rate_search_master_products_trgm_index_and_numeric_guard.sql new file mode 100644 index 00000000..80a94590 --- /dev/null +++ b/supabase/migrations/20261104120000_rate_search_master_products_trgm_index_and_numeric_guard.sql @@ -0,0 +1,206 @@ +-- rate.search_master_products: let the text search use a trigram index, and +-- stop a malformed spec number from failing the whole search (DOS/DOS.Me#1262, +-- from the Gemini review of DOS/DOS.Me#1258). +-- +-- 1. Index use. The search matched +-- concat_ws(' ', brand, name, model, model_key, variant_name, chip, chipset) +-- but idx_master_products_search_trgm indexes only +-- brand || ' ' || name || ' ' || model || ' ' || model_key, +-- and concat_ws is STABLE, so no index can be built on it. Even a matching +-- index would stay out of the plan: the filter is +-- "p_search IS NULL OR btrim(p_search) = '' OR ", and a LANGUAGE sql +-- function that is not inlined (SET search_path prevents inlining) is +-- planned on PostgreSQL 17 with its arguments as unknown parameters, so the +-- OR is never simplified, and an OR with a non-indexable arm cannot use an +-- index. +-- This migration: +-- * adds idx_master_products_search_full_trgm on the same seven fields, +-- written with || and COALESCE (immutable). brand, name, model and +-- model_key are NOT NULL, so the text is the one concat_ws built and +-- a search matches exactly the rows it matched before; +-- * rewrites the function in plpgsql with SET plan_cache_mode = +-- force_custom_plan, so every call is planned with its actual +-- arguments: the IS NULL / empty-string arms fold away and the match is +-- a plain ILIKE on the indexed expression. +-- The function stays SECURITY INVOKER with SET search_path = '', the same +-- signature, return type, filters, ordering, paging and grants. +-- +-- 2. Numeric cast. The refresh-rate, fast-charging and screen-size filters +-- strip every character but digits and dots from crawled spec text and cast +-- the rest to NUMERIC. A value such as '120.5.6' or '.' then raised +-- "invalid input syntax for type numeric" and failed the search for every +-- caller using that filter. The stripped text is now cast only when it is a +-- valid number (digits with at most one dot); otherwise it counts as +-- unknown, the same as an empty value did before. Every value that cast +-- successfully before gives the same number. +-- +-- rate.master_products is small (37 rows on Production on 2026-11-01), so a +-- plain CREATE INDEX (which a migration transaction requires; CONCURRENTLY +-- cannot run in one) holds its lock only briefly. lock_timeout makes it fail +-- fast instead of queueing behind a long transaction. Idempotent: CREATE INDEX +-- IF NOT EXISTS and CREATE OR REPLACE FUNCTION. + +SET lock_timeout = '5s'; + +-- pg_trgm lives in `sign` on Production and in `extensions` on fresh branches: +-- resolve its schema, as 20261014231500 does for the existing trgm indexes. +DO $rate_master_products_search_full_trgm$ +DECLARE + v_trgm_schema text; +BEGIN + SELECT n.nspname INTO v_trgm_schema + FROM pg_extension e + JOIN pg_namespace n ON n.oid = e.extnamespace + WHERE e.extname = 'pg_trgm'; + + IF v_trgm_schema IS NULL THEN + RAISE EXCEPTION 'rate schema: pg_trgm extension is not installed'; + END IF; + + -- Keep this expression identical to the one in rate.search_master_products + -- below, or the planner cannot match the index. + EXECUTE format( + 'CREATE INDEX IF NOT EXISTS idx_master_products_search_full_trgm + ON rate.master_products + USING gin ( + ( + COALESCE(brand, '''') || '' '' || + COALESCE(name, '''') || '' '' || + COALESCE(model, '''') || '' '' || + COALESCE(model_key, '''') || '' '' || + COALESCE(variant_name, '''') || '' '' || + COALESCE(key_specs ->> ''chip'', '''') || '' '' || + COALESCE(specifications #>> ''{hardware,chipset}'', '''') + ) %I.gin_trgm_ops + )', + v_trgm_schema + ); +END +$rate_master_products_search_full_trgm$; + +CREATE OR REPLACE FUNCTION rate.search_master_products( + p_category_id UUID DEFAULT NULL, + p_brands TEXT[] DEFAULT NULL, + p_search TEXT DEFAULT NULL, + p_min_price NUMERIC DEFAULT NULL, + p_max_price NUMERIC DEFAULT NULL, + p_min_rating NUMERIC DEFAULT NULL, + p_ram_gb INTEGER[] DEFAULT NULL, + p_storage_gb INTEGER[] DEFAULT NULL, + p_min_battery_mah INTEGER DEFAULT NULL, + p_min_refresh_rate_hz INTEGER DEFAULT NULL, + p_min_fast_charging_w INTEGER DEFAULT NULL, + p_has_5g BOOLEAN DEFAULT NULL, + p_has_nfc BOOLEAN DEFAULT NULL, + p_max_screen_size_inch NUMERIC DEFAULT NULL, + p_sort TEXT DEFAULT 'default', + p_limit INTEGER DEFAULT 200, + p_offset INTEGER DEFAULT 0 +) +RETURNS SETOF rate.master_products +LANGUAGE plpgsql +STABLE +SECURITY INVOKER +SET search_path = '' +SET plan_cache_mode = 'force_custom_plan' +AS $$ +BEGIN + RETURN QUERY + SELECT m.* + FROM rate.master_products m + CROSS JOIN LATERAL ( + SELECT + pg_catalog.regexp_replace(COALESCE(m.specifications #>> '{display,refresh_rate_hz}', ''), '[^0-9.]', '', 'g') AS refresh_rate_hz, + pg_catalog.regexp_replace(COALESCE(m.specifications #>> '{battery,fast_charging_w}', ''), '[^0-9.]', '', 'g') AS fast_charging_w, + pg_catalog.regexp_replace(COALESCE(m.specifications #>> '{display,size_inch}', ''), '[^0-9.]', '', 'g') AS size_inch + ) spec + WHERE m.status = 'published' + AND (p_category_id IS NULL OR m.category_id = p_category_id) + AND ( + p_brands IS NULL + OR cardinality(p_brands) = 0 + OR pg_catalog.lower(pg_catalog.btrim(m.brand)) = ANY ( + SELECT pg_catalog.lower(pg_catalog.btrim(b)) FROM unnest(p_brands) AS b + ) + ) + AND ( + p_search IS NULL + OR pg_catalog.btrim(p_search) = '' + OR ( + -- Same expression as idx_master_products_search_full_trgm. + ( + COALESCE(m.brand, '') || ' ' || + COALESCE(m.name, '') || ' ' || + COALESCE(m.model, '') || ' ' || + COALESCE(m.model_key, '') || ' ' || + COALESCE(m.variant_name, '') || ' ' || + COALESCE(m.key_specs ->> 'chip', '') || ' ' || + COALESCE(m.specifications #>> '{hardware,chipset}', '') + ) ILIKE '%' || pg_catalog.regexp_replace(pg_catalog.btrim(p_search), '([%_\\])', '\\\1', 'g') || '%' ESCAPE '\' + ) + ) + AND (p_min_price IS NULL OR p_min_price <= 0 OR m.min_price >= p_min_price) + AND (p_max_price IS NULL OR p_max_price <= 0 OR m.min_price <= p_max_price) + AND ( + p_min_rating IS NULL + OR p_min_rating <= 0 + OR COALESCE( + NULLIF(m.user_rating_weighted, 0), + NULLIF(m.expert_rating, 0), + m.average_rating, + 0 + ) >= p_min_rating + ) + AND (p_ram_gb IS NULL OR cardinality(p_ram_gb) = 0 OR m.ram_gb = ANY (p_ram_gb)) + AND (p_storage_gb IS NULL OR cardinality(p_storage_gb) = 0 OR m.storage_gb = ANY (p_storage_gb)) + AND (p_min_battery_mah IS NULL OR p_min_battery_mah <= 0 OR m.battery_mah >= p_min_battery_mah) + AND ( + p_min_refresh_rate_hz IS NULL + OR p_min_refresh_rate_hz <= 0 + OR COALESCE( + CASE WHEN spec.refresh_rate_hz ~ '^([0-9]+\.?[0-9]*|\.[0-9]+)$' THEN spec.refresh_rate_hz::NUMERIC END, + 0 + ) >= p_min_refresh_rate_hz + OR COALESCE(m.key_specs ->> 'screen', '') ILIKE '%120hz%' + OR COALESCE(m.key_specs ->> 'screen', '') ILIKE '%144hz%' + ) + AND ( + p_min_fast_charging_w IS NULL + OR p_min_fast_charging_w <= 0 + OR COALESCE( + CASE WHEN spec.fast_charging_w ~ '^([0-9]+\.?[0-9]*|\.[0-9]+)$' THEN spec.fast_charging_w::NUMERIC END, + 0 + ) >= p_min_fast_charging_w + ) + AND (p_has_5g IS NULL OR p_has_5g = false OR m.has_5g = true) + AND ( + p_has_nfc IS NULL + OR p_has_nfc = false + OR (m.specifications -> 'connectivity' ->> 'nfc') = 'true' + ) + AND ( + p_max_screen_size_inch IS NULL + OR p_max_screen_size_inch <= 0 + OR COALESCE( + m.screen_size_inch, + CASE WHEN spec.size_inch ~ '^([0-9]+\.?[0-9]*|\.[0-9]+)$' THEN spec.size_inch::NUMERIC END + ) <= p_max_screen_size_inch + ) + ORDER BY + CASE WHEN p_sort = 'price_asc' THEN m.min_price END ASC NULLS LAST, + CASE WHEN p_sort = 'price_desc' THEN m.min_price END DESC NULLS LAST, + CASE WHEN p_sort = 'merchants_desc' THEN m.merchant_count END DESC NULLS LAST, + CASE WHEN p_sort = 'rating_desc' THEN COALESCE(NULLIF(m.user_rating_weighted, 0), m.average_rating, 0) END DESC NULLS LAST, + m.created_at DESC, + m.id ASC + LIMIT GREATEST(1, LEAST(COALESCE(p_limit, 200), 500)) + OFFSET GREATEST(0, COALESCE(p_offset, 0)); +END; +$$; + +GRANT EXECUTE ON FUNCTION rate.search_master_products( + UUID, TEXT[], TEXT, NUMERIC, NUMERIC, NUMERIC, INTEGER[], INTEGER[], + INTEGER, INTEGER, INTEGER, BOOLEAN, BOOLEAN, NUMERIC, TEXT, INTEGER, INTEGER +) TO anon, authenticated, service_role; + +RESET lock_timeout; diff --git a/supabase/migrations/20261104140000_rate_search_master_products_rating_score.sql b/supabase/migrations/20261104140000_rate_search_master_products_rating_score.sql new file mode 100644 index 00000000..fa5abd5f --- /dev/null +++ b/supabase/migrations/20261104140000_rate_search_master_products_rating_score.sql @@ -0,0 +1,176 @@ +-- Migration: 20261104140000_rate_search_master_products_rating_score.sql +-- Purpose (DOS/Rate#1545, phase P4 of DOS/Rate#1531): rate.search_master_products sorts +-- ("highest rated") and filters (minimum rating) on the rating every page shows: rating_score, +-- else the weighted user rating, else the marketplace average. It replaces DOS/DOS.Me#1497's +-- function (20261104120000_rate_search_master_products_trgm_index_and_numeric_guard.sql) with +-- only the rating expressions changed, so it must apply after it. +-- The function is copied unchanged from the reviewed DOS/Rate proposal +-- Docs/Proposals/2026-11-02-dos-me-search-rating-score.sql (DOS/Rate#1574); only the header +-- comments differ. +-- Verify: scripts/ops/rate-search-rating-score-verify.sql (V1 to V5 return 0). +-- Rollback: scripts/ops/rate-search-rating-score-rollback.sql (#1497's function). +-- +-- rate.search_master_products: the catalog's "highest rated" order (p_sort = 'rating_desc') and +-- its minimum rating filter (p_min_rating) read the one rating every page shows (design D3): +-- 1. master_products.rating_score (P2, the stored Bayesian score), when it is above 0 and at +-- most 5; +-- 2. else user_rating_weighted, then average_rating, each read as the web reads them +-- (normalizeCompareFiveStarRating: a value above 0 and at most 100, a 0-10 or 0-100 scale +-- brought to 0-5, rounded to 2 decimals). +-- This is resolveCompareEffectiveUserRating in apps/web/src/lib/rate-domain.ts. Before, the order +-- read user_rating_weighted then average_rating, and the filter also fell back to expert_rating, +-- which no page shows as the product's rating; so the catalog, `?rating=` and the leaderboard's +-- top-100 load could disagree with the rating on screen. The filter compares the rating at one +-- decimal, as the pages show it (productRatingScore5); the order uses it unrounded. A product +-- with none of the three sorts last and never passes a minimum rating, as before. +-- +-- Needs, in this order: +-- * P2 (20261101120000_rate_criteria_ratings.sql): the rating_score column; +-- * DOS/DOS.Me#1497 (20261104120000_rate_search_master_products_trgm_index_and_numeric_guard): +-- this function body is #1497's with only the two rating expressions changed. Applied +-- before #1497, #1497 would put the old rating expressions back. +-- Same signature, return type, SECURITY INVOKER, search_path, plan_cache_mode, filters, paging and +-- grants as #1497. No table, index or data change. Idempotent: CREATE OR REPLACE FUNCTION. + +CREATE OR REPLACE FUNCTION rate.search_master_products( + p_category_id UUID DEFAULT NULL, + p_brands TEXT[] DEFAULT NULL, + p_search TEXT DEFAULT NULL, + p_min_price NUMERIC DEFAULT NULL, + p_max_price NUMERIC DEFAULT NULL, + p_min_rating NUMERIC DEFAULT NULL, + p_ram_gb INTEGER[] DEFAULT NULL, + p_storage_gb INTEGER[] DEFAULT NULL, + p_min_battery_mah INTEGER DEFAULT NULL, + p_min_refresh_rate_hz INTEGER DEFAULT NULL, + p_min_fast_charging_w INTEGER DEFAULT NULL, + p_has_5g BOOLEAN DEFAULT NULL, + p_has_nfc BOOLEAN DEFAULT NULL, + p_max_screen_size_inch NUMERIC DEFAULT NULL, + p_sort TEXT DEFAULT 'default', + p_limit INTEGER DEFAULT 200, + p_offset INTEGER DEFAULT 0 +) +RETURNS SETOF rate.master_products +LANGUAGE plpgsql +STABLE +SECURITY INVOKER +SET search_path = '' +SET plan_cache_mode = 'force_custom_plan' +AS $$ +BEGIN + RETURN QUERY + SELECT m.* + FROM rate.master_products m + CROSS JOIN LATERAL ( + SELECT + pg_catalog.regexp_replace(COALESCE(m.specifications #>> '{display,refresh_rate_hz}', ''), '[^0-9.]', '', 'g') AS refresh_rate_hz, + pg_catalog.regexp_replace(COALESCE(m.specifications #>> '{battery,fast_charging_w}', ''), '[^0-9.]', '', 'g') AS fast_charging_w, + pg_catalog.regexp_replace(COALESCE(m.specifications #>> '{display,size_inch}', ''), '[^0-9.]', '', 'g') AS size_inch + ) spec + -- The rating the pages show (DOS/Rate#1545): rating_score, else the weighted user rating, else + -- the marketplace average, each on 0-5 as resolveCompareEffectiveUserRating reads it. NULL when + -- the product has none. + CROSS JOIN LATERAL ( + SELECT CASE + WHEN m.rating_score > 0 AND m.rating_score <= 5 THEN m.rating_score + ELSE COALESCE( + CASE WHEN m.user_rating_weighted > 0 AND m.user_rating_weighted <= 100 THEN pg_catalog.round( + CASE + WHEN m.user_rating_weighted > 10 THEN m.user_rating_weighted / 20 + WHEN m.user_rating_weighted > 5 THEN m.user_rating_weighted / 2 + ELSE m.user_rating_weighted + END, 2) END, + CASE WHEN m.average_rating > 0 AND m.average_rating <= 100 THEN pg_catalog.round( + CASE + WHEN m.average_rating > 10 THEN m.average_rating / 20 + WHEN m.average_rating > 5 THEN m.average_rating / 2 + ELSE m.average_rating + END, 2) END + ) + END AS rating_5 + ) shown + WHERE m.status = 'published' + AND (p_category_id IS NULL OR m.category_id = p_category_id) + AND ( + p_brands IS NULL + OR cardinality(p_brands) = 0 + OR pg_catalog.lower(pg_catalog.btrim(m.brand)) = ANY ( + SELECT pg_catalog.lower(pg_catalog.btrim(b)) FROM unnest(p_brands) AS b + ) + ) + AND ( + p_search IS NULL + OR pg_catalog.btrim(p_search) = '' + OR ( + -- Same expression as idx_master_products_search_full_trgm. + ( + COALESCE(m.brand, '') || ' ' || + COALESCE(m.name, '') || ' ' || + COALESCE(m.model, '') || ' ' || + COALESCE(m.model_key, '') || ' ' || + COALESCE(m.variant_name, '') || ' ' || + COALESCE(m.key_specs ->> 'chip', '') || ' ' || + COALESCE(m.specifications #>> '{hardware,chipset}', '') + ) ILIKE '%' || pg_catalog.regexp_replace(pg_catalog.btrim(p_search), '([%_\\])', '\\\1', 'g') || '%' ESCAPE '\' + ) + ) + AND (p_min_price IS NULL OR p_min_price <= 0 OR m.min_price >= p_min_price) + AND (p_max_price IS NULL OR p_max_price <= 0 OR m.min_price <= p_max_price) + AND ( + p_min_rating IS NULL + OR p_min_rating <= 0 + -- At one decimal, as the pages show it: 4.46 shows 4.5 and passes 4.5. + OR pg_catalog.round(COALESCE(shown.rating_5, 0), 1) >= p_min_rating + ) + AND (p_ram_gb IS NULL OR cardinality(p_ram_gb) = 0 OR m.ram_gb = ANY (p_ram_gb)) + AND (p_storage_gb IS NULL OR cardinality(p_storage_gb) = 0 OR m.storage_gb = ANY (p_storage_gb)) + AND (p_min_battery_mah IS NULL OR p_min_battery_mah <= 0 OR m.battery_mah >= p_min_battery_mah) + AND ( + p_min_refresh_rate_hz IS NULL + OR p_min_refresh_rate_hz <= 0 + OR COALESCE( + CASE WHEN spec.refresh_rate_hz ~ '^([0-9]+\.?[0-9]*|\.[0-9]+)$' THEN spec.refresh_rate_hz::NUMERIC END, + 0 + ) >= p_min_refresh_rate_hz + OR COALESCE(m.key_specs ->> 'screen', '') ILIKE '%120hz%' + OR COALESCE(m.key_specs ->> 'screen', '') ILIKE '%144hz%' + ) + AND ( + p_min_fast_charging_w IS NULL + OR p_min_fast_charging_w <= 0 + OR COALESCE( + CASE WHEN spec.fast_charging_w ~ '^([0-9]+\.?[0-9]*|\.[0-9]+)$' THEN spec.fast_charging_w::NUMERIC END, + 0 + ) >= p_min_fast_charging_w + ) + AND (p_has_5g IS NULL OR p_has_5g = false OR m.has_5g = true) + AND ( + p_has_nfc IS NULL + OR p_has_nfc = false + OR (m.specifications -> 'connectivity' ->> 'nfc') = 'true' + ) + AND ( + p_max_screen_size_inch IS NULL + OR p_max_screen_size_inch <= 0 + OR COALESCE( + m.screen_size_inch, + CASE WHEN spec.size_inch ~ '^([0-9]+\.?[0-9]*|\.[0-9]+)$' THEN spec.size_inch::NUMERIC END + ) <= p_max_screen_size_inch + ) + ORDER BY + CASE WHEN p_sort = 'price_asc' THEN m.min_price END ASC NULLS LAST, + CASE WHEN p_sort = 'price_desc' THEN m.min_price END DESC NULLS LAST, + CASE WHEN p_sort = 'merchants_desc' THEN m.merchant_count END DESC NULLS LAST, + CASE WHEN p_sort = 'rating_desc' THEN COALESCE(shown.rating_5, 0) END DESC NULLS LAST, + m.created_at DESC, + m.id ASC + LIMIT GREATEST(1, LEAST(COALESCE(p_limit, 200), 500)) + OFFSET GREATEST(0, COALESCE(p_offset, 0)); +END; +$$; + +GRANT EXECUTE ON FUNCTION rate.search_master_products( + UUID, TEXT[], TEXT, NUMERIC, NUMERIC, NUMERIC, INTEGER[], INTEGER[], + INTEGER, INTEGER, INTEGER, BOOLEAN, BOOLEAN, NUMERIC, TEXT, INTEGER, INTEGER +) TO anon, authenticated, service_role;