diff --git a/app/src/api/billFeed.ts b/app/src/api/billFeed.ts index 39db3083e..02e52df6d 100644 --- a/app/src/api/billFeed.ts +++ b/app/src/api/billFeed.ts @@ -189,6 +189,13 @@ function totalRows(response: Response): number | undefined { async function trackerSelect( table: string, select: string, + /** + * A unique column that puts the rows in one fixed order ("bill_id"). + * With no order, the database can give each page its rows in another + * order, so the pages repeat some rows and drop others, and different + * ones on each load. + */ + order: string, /** A PostgREST row filter, e.g. "id=eq.ks-hb2620". */ filter?: string ): Promise { @@ -199,7 +206,7 @@ async function trackerSelect( // Read page by page: one request stops at the cap and drops the rest. const rows: any[] = []; for (;;) { - const query = `select=${select}${filter ? `&${filter}` : ''}&limit=${TRACKER_PAGE_SIZE}&offset=${rows.length}`; + const query = `select=${select}${filter ? `&${filter}` : ''}&order=${order}&limit=${TRACKER_PAGE_SIZE}&offset=${rows.length}`; const response = await fetch(`${config.url}/rest/v1/${table}?${query}`, { headers: { apikey: config.anonKey, @@ -500,6 +507,7 @@ export async function fetchTrackerBillDistricts( const rows = await trackerSelect( 'reform_impacts', 'district_impacts', + 'id', `id=eq.${encodeURIComponent(billId)}` ); const raw = rows?.[0]?.district_impacts; @@ -560,6 +568,58 @@ function normalizeBillNumber(value: string): string { const processedBillKey = (state: string, billNumber: string) => `${state.toUpperCase()}:${normalizeBillNumber(billNumber)}`; +/** A status row's last action day ("2026-03-04"), or "" when it has none. */ +const lastActionDay = (bill: any): string => { + const day = typeof bill.last_action_date === 'string' ? bill.last_action_date.slice(0, 10) : ''; + return /^\d{4}-\d{2}-\d{2}$/.test(day) ? day : ''; +}; + +/** + * Whether a status row came from LegiScan, read from its link: the + * OpenStates and Congress monitors store their own sites' links in the + * same column, and the row's source column defaults to "legiscan". + */ +const isLegiscanRow = (bill: any): boolean => + /^https?:\/\/(www\.)?legiscan\.com\//i.test(String(bill.legiscan_url ?? '')); + +/** A status row's bill id as a number, lowest when it has none. */ +const statusRowId = (bill: any): number => { + const id = Number(bill.bill_id ?? NaN); + return Number.isFinite(id) ? id : -Infinity; +}; + +const statusRowText = (bill: any) => + JSON.stringify([ + bill.bill_id, + bill.state, + bill.bill_number, + bill.status, + bill.legiscan_url, + bill.last_action_date, + ]); + +const ascending = (a: number | string, b: number | string) => (a < b ? -1 : a > b ? 1 : 0); + +/** + * Orders two status rows for one bill number, the preferred row last. + * The tracker can hold a bill twice under two spellings of its number: + * LegiScan's (NY "S04487") and OpenStates' ("S 4487"). The row with the + * later last action wins. On the same day, or with no dates, the LegiScan + * row wins: it names the stage more finely ("In Committee", where + * OpenStates says "Introduced") and links to LegiScan. The bill id and + * then the row's contents only fix the order of what is left. They do + * not date a bill: OpenStates and Congress ids are hashes, all above + * LegiScan's. The same row wins in any read order. + */ +function compareStatusRows(a: any, b: any): number { + return ( + ascending(lastActionDay(a), lastActionDay(b)) || + ascending(Number(isLegiscanRow(a)), Number(isLegiscanRow(b))) || + ascending(statusRowId(a), statusRowId(b)) || + ascending(statusRowText(a), statusRowText(b)) + ); +} + /** * The tracker's status row (processed_bills) for a research entry. The * entry's LegiScan link (legiscan_bill_id) names the row when it is set. @@ -595,19 +655,25 @@ function processedBillFor( export async function fetchTrackerBills(): Promise { const [research, impacts, processed, validations, baselines] = await Promise.all([ // Titled as the tracker shows them: an entry with no title reads as its bill number. - trackerSelect('research', '*').then((rows) => rows?.map(withTrackerTitle) ?? null), + trackerSelect('research', '*', 'id').then((rows) => rows?.map(withTrackerTitle) ?? null), trackerSelect( 'reform_impacts', - 'id,reform_params,provisions,computed,budgetary_impact,poverty_impact,child_poverty_impact,winners_losers,decile_impact,policyengine_us_version,dataset_name,dataset_version,computed_at' + 'id,reform_params,provisions,computed,budgetary_impact,poverty_impact,child_poverty_impact,winners_losers,decile_impact,policyengine_us_version,dataset_name,dataset_version,computed_at', + 'id' + ), + trackerSelect( + 'processed_bills', + 'bill_id,state,bill_number,status,legiscan_url,last_action_date', + 'bill_id' ), - trackerSelect('processed_bills', 'bill_id,state,bill_number,status,legiscan_url'), trackerSelect( 'validation_metadata', - 'id,fiscal_note_source,fiscal_note_url,fiscal_note_estimate,pe_estimate,target_range_low,target_range_high,within_range,difference_from_fiscal_note_pct,discrepancy_explanation,external_analyses,verification,verified_at,validated_against' + 'id,fiscal_note_source,fiscal_note_url,fiscal_note_estimate,pe_estimate,target_range_low,target_range_high,within_range,difference_from_fiscal_note_pct,discrepancy_explanation,external_analyses,verification,verified_at,validated_against', + 'id' ), // The law each bill was compared with. Read on its own, so a tracker // without the column yet still serves the rest of the feed. - trackerSelect('reform_impacts', 'id,baseline_params').catch(() => null), + trackerSelect('reform_impacts', 'id,baseline_params', 'id').catch(() => null), ]); if (!research) { return null; @@ -621,12 +687,15 @@ export async function fetchTrackerBills(): Promise { bill.bill_id === null || bill.bill_id === undefined ? [] : [[String(bill.bill_id), bill]] ) ); - const processedByKey = new Map( - (processed ?? []).map((bill) => [ - processedBillKey(String(bill.state), String(bill.bill_number)), - bill, - ]) - ); + // Two rows can share a bill number's key: the preferred one holds it. + const processedByKey = new Map(); + for (const bill of processed ?? []) { + const key = processedBillKey(String(bill.state), String(bill.bill_number)); + const held = processedByKey.get(key); + if (!held || compareStatusRows(held, bill) < 0) { + processedByKey.set(key, bill); + } + } return research .filter((record) => isBillRecord(record)) diff --git a/app/src/tests/fixtures/api/billFeedMocks.ts b/app/src/tests/fixtures/api/billFeedMocks.ts index 1c4dc74bb..ec2fbf2fb 100644 --- a/app/src/tests/fixtures/api/billFeedMocks.ts +++ b/app/src/tests/fixtures/api/billFeedMocks.ts @@ -45,3 +45,171 @@ export const NET_COST_DRIFT_ROWS: TrackerTableRows = { }, ], }; + +/** The most rows one tracker read returns, as the Supabase cap sets it. */ +export const TRACKER_PAGE_LIMIT = 1000; + +/** The tracker's bill-status table holds more rows than one read returns. */ +export const STATUS_TABLE_ROWS = 1418; + +/** The first column a PostgREST `order` names ("bill_id" in "bill_id.asc"). */ +const orderColumn = (order: string) => order.split(',')[0].split('.')[0]; + +const byColumn = (column: string) => (a: any, b: any) => + a[column] < b[column] ? -1 : a[column] > b[column] ? 1 : 0; + +/** + * A fetch that serves each tracker table a page at a time, as PostgREST + * does. A read that names an `order` gets its rows in that order. A read + * with none has no fixed row order: as on the live tracker, every page + * starts from the table's first rows again, so a later page repeats rows + * already read and the rest never arrive. + */ +export const mockPagedTrackerFetch = (rows: TrackerTableRows) => + vi.fn(async (url: string) => { + const { pathname, searchParams } = new URL(url); + const all = rows[pathname.split('/').pop() ?? ''] ?? []; + const order = searchParams.get('order'); + const offset = Number(searchParams.get('offset') ?? 0); + const limit = Number(searchParams.get('limit') ?? all.length); + const count = Math.max(0, Math.min(limit, all.length - offset)); + const page = order + ? [...all].sort(byColumn(orderColumn(order))).slice(offset, offset + count) + : all.slice(0, count); + return { + ok: true, + headers: new Headers({ + 'content-range': `${offset}-${offset + page.length - 1}/${all.length}`, + }), + json: async () => page, + }; + }); + +/** WV SB 392, whose one status row is the bill-status table's last. */ +export const LATE_STATUS_BILL = { + id: 'wv-sb392', + state: 'WV', + type: 'bill', + title: 'WV SB 392', +} as const; +export const LATE_STATUS = { + status: 'Signed into Law', + legiscanUrl: 'https://legiscan.com/WV/bill/SB392/2026', +} as const; + +/** A full bill-status table with WV SB 392's row past the first page. */ +export const LATE_STATUS_ROWS: TrackerTableRows = { + research: [LATE_STATUS_BILL], + processed_bills: Array.from({ length: STATUS_TABLE_ROWS }, (_, index) => + index === STATUS_TABLE_ROWS - 1 + ? { + bill_id: 2_000_000 + index, + state: 'WV', + bill_number: 'SB392', + status: LATE_STATUS.status, + legiscan_url: LATE_STATUS.legiscanUrl, + } + : { + bill_id: 2_000_000 + index, + state: 'OK', + bill_number: `HB${1000 + index}`, + status: 'Introduced', + } + ), +}; + +/** Bills enough that every tracker table runs past one page. */ +export const MANY_BILLS = TRACKER_PAGE_LIMIT + 1; +export const MANY_BILLS_STATUS = 'In Committee'; +/** The rate each of those bills changes, and the law it was compared with. */ +const MANY_BILLS_RATE_PATH = 'gov.states.ok.tax.income.rate'; +export const MANY_BILLS_BASELINE_RATE = 0.0475; + +const MANY_BILL_NUMBERS = Array.from({ length: MANY_BILLS }, (_, index) => 1000 + index); + +/** Every table past one page: each bill has its entry, run, check and status row. */ +export const MANY_BILL_ROWS: TrackerTableRows = { + research: MANY_BILL_NUMBERS.map((number) => ({ + id: `ok-hb${number}`, + state: 'OK', + type: 'bill', + title: `OK HB ${number}`, + })), + // One row serves both reform_impacts reads: the run and its baseline. + reform_impacts: MANY_BILL_NUMBERS.map((number) => ({ + id: `ok-hb${number}`, + computed: true, + budgetary_impact: { stateRevenueImpact: -number * 1000 }, + reform_params: { [MANY_BILLS_RATE_PATH]: 0.04 }, + baseline_params: { [MANY_BILLS_RATE_PATH]: MANY_BILLS_BASELINE_RATE }, + })), + validation_metadata: MANY_BILL_NUMBERS.map((number) => ({ + id: `ok-hb${number}`, + pe_estimate: -number * 1000, + fiscal_note_estimate: -number * 1000, + within_range: true, + })), + processed_bills: MANY_BILL_NUMBERS.map((number) => ({ + bill_id: 3_000_000 + number, + state: 'OK', + bill_number: `HB ${number}`, + status: MANY_BILLS_STATUS, + })), +}; + +/** NY S 4487, held twice in the status table under two spellings of its number. */ +export const TWICE_FILED_BILL = { + id: 'ny-s-4487', + state: 'NY', + type: 'bill', + title: 'NY S 4487', +} as const; +/** LegiScan's row: a LegiScan bill id (about 2 million) and its finer stage. */ +export const TWICE_FILED_LEGISCAN = { + bill_id: 1_987_654, + state: 'NY', + bill_number: 'S04487', + status: 'In Committee', + legiscan_url: 'https://legiscan.com/NY/bill/S04487/2025', +} as const; +/** OpenStates' row: the monitor hashes its bill id to 10,000,000 or more. */ +export const TWICE_FILED_OPENSTATES = { + bill_id: 47_123_456, + state: 'NY', + bill_number: 'S 4487', + status: 'Introduced', + legiscan_url: 'https://openstates.org/ny/bills/2025-2026/S4487/', +} as const; + +/** The two rows' last action dates, and the row that keeps the bill number. */ +export const TWICE_FILED_CASES = [ + { + when: 'the LegiScan row acted last', + winnerName: 'LegiScan', + legiscanDate: '2025-05-02', + openstatesDate: '2025-03-10', + winner: TWICE_FILED_LEGISCAN, + }, + { + when: 'the OpenStates row acted last', + winnerName: 'OpenStates', + legiscanDate: '2025-03-10', + openstatesDate: '2025-05-02', + winner: TWICE_FILED_OPENSTATES, + }, + { + // A timestamp still reads as its day. + when: 'both acted the same day', + winnerName: 'LegiScan', + legiscanDate: '2025-03-10', + openstatesDate: '2025-03-10T14:30:00+00:00', + winner: TWICE_FILED_LEGISCAN, + }, + { + when: 'neither has a date', + winnerName: 'LegiScan', + legiscanDate: null, + openstatesDate: null, + winner: TWICE_FILED_LEGISCAN, + }, +]; diff --git a/app/src/tests/unit/api/billFeed.test.ts b/app/src/tests/unit/api/billFeed.test.ts index fd0c4da9b..c133571ef 100644 --- a/app/src/tests/unit/api/billFeed.test.ts +++ b/app/src/tests/unit/api/billFeed.test.ts @@ -9,10 +9,21 @@ import { } from '@/api/billFeed'; import { CURRENT_YEAR } from '@/constants'; import { + LATE_STATUS, + LATE_STATUS_ROWS, + MANY_BILL_ROWS, + MANY_BILLS, + MANY_BILLS_BASELINE_RATE, + MANY_BILLS_STATUS, + mockPagedTrackerFetch, mockTrackerFetch, NET_COST_DRIFT_ROWS, NET_COST_ONLY_ROWS, TRACKER_NAMING_ROWS, + TWICE_FILED_BILL, + TWICE_FILED_CASES, + TWICE_FILED_LEGISCAN, + TWICE_FILED_OPENSTATES, } from '@/tests/fixtures/api/billFeedMocks'; import { TRACKER_HOME } from '@/tests/fixtures/libs/flagship/billTrackerMocks'; @@ -300,6 +311,70 @@ describe('tracker reads', () => { '1000', ]); }); + + test('given a status row past the first page then every load finds its stage and link', async () => { + vi.stubEnv('NEXT_PUBLIC_TRACKER_SUPABASE_URL', 'https://tracker.example.supabase.co'); + vi.stubEnv('NEXT_PUBLIC_TRACKER_SUPABASE_ANON_KEY', 'anon-key'); + // Pages read with no fixed order repeat rows and skip the rest. + vi.stubGlobal('fetch', mockPagedTrackerFetch(LATE_STATUS_ROWS)); + + const [bill] = (await fetchTrackerBills())!; + + expect([bill.status, bill.legiscanUrl]).toEqual([LATE_STATUS.status, LATE_STATUS.legiscanUrl]); + }); + + test('given every table past one page then each bill reads once, with its run, baseline, check and stage', async () => { + vi.stubEnv('NEXT_PUBLIC_TRACKER_SUPABASE_URL', 'https://tracker.example.supabase.co'); + vi.stubEnv('NEXT_PUBLIC_TRACKER_SUPABASE_ANON_KEY', 'anon-key'); + vi.stubGlobal('fetch', mockPagedTrackerFetch(MANY_BILL_ROWS)); + + const bills = (await fetchTrackerBills())!; + + const billsWith = (has: (bill: (typeof bills)[number]) => boolean) => + new Set(bills.filter(has).map((bill) => bill.id)).size; + expect({ + listed: bills.length, + unique: billsWith(() => true), + withRun: billsWith((bill) => bill.impacts?.revenue !== undefined), + withBaseline: billsWith( + (bill) => bill.provisions[0]?.baselineIntervals?.[0]?.value === MANY_BILLS_BASELINE_RATE + ), + withCheck: billsWith((bill) => bill.validation?.withinRange === true), + withStage: billsWith((bill) => bill.status === MANY_BILLS_STATUS), + }).toEqual({ + listed: MANY_BILLS, + unique: MANY_BILLS, + withRun: MANY_BILLS, + withBaseline: MANY_BILLS, + withCheck: MANY_BILLS, + withStage: MANY_BILLS, + }); + }); + + test.each(TWICE_FILED_CASES)( + 'given two status rows under one bill number when $when then the $winnerName row wins in any read order', + async ({ legiscanDate, openstatesDate, winner }) => { + vi.stubEnv('NEXT_PUBLIC_TRACKER_SUPABASE_URL', 'https://tracker.example.supabase.co'); + vi.stubEnv('NEXT_PUBLIC_TRACKER_SUPABASE_ANON_KEY', 'anon-key'); + const legiscan = { ...TWICE_FILED_LEGISCAN, last_action_date: legiscanDate }; + const openstates = { ...TWICE_FILED_OPENSTATES, last_action_date: openstatesDate }; + const loads = []; + for (const processed of [ + [legiscan, openstates], + [openstates, legiscan], + ]) { + vi.stubGlobal( + 'fetch', + mockTrackerFetch({ research: [TWICE_FILED_BILL], processed_bills: processed }) + ); + const [bill] = (await fetchTrackerBills())!; + loads.push([bill.status, bill.legiscanUrl]); + } + + const won = [winner.status, winner.legiscan_url]; + expect(loads).toEqual([won, won]); + } + ); }); describe('fetchTrackerBills', () => {