From 219cb56eb005e94ffa39329c2b24eb632a623d63 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 9 Oct 2026 13:58:16 +0100 Subject: [PATCH 1/2] Split random-access benchmarks into NVMe and S3 sections Signed-off-by: Joe Isaacs --- web/lib/benchmark-docs.test.ts | 2 +- web/lib/data-cache.test.ts | 4 +-- web/lib/data-cache.ts | 2 +- web/lib/descriptions.test.ts | 6 ++++ web/lib/descriptions.ts | 3 ++ web/lib/groups.test.ts | 63 ++++++++++++++++++++++++++++------ web/lib/queries.test.ts | 21 +++++++----- web/lib/queries.ts | 29 +++++++++++----- web/lib/random-access.test.ts | 21 ++++++++++++ web/lib/random-access.ts | 12 +++++++ web/lib/slug.test.ts | 14 +++++++- web/lib/slug.ts | 16 +++++++-- web/lib/summary.test.ts | 48 +++++++++++++++++++++----- web/lib/summary.ts | 12 ++++--- 14 files changed, 206 insertions(+), 47 deletions(-) create mode 100644 web/lib/random-access.test.ts create mode 100644 web/lib/random-access.ts diff --git a/web/lib/benchmark-docs.test.ts b/web/lib/benchmark-docs.test.ts index e4b6dd9..51a8fbd 100644 --- a/web/lib/benchmark-docs.test.ts +++ b/web/lib/benchmark-docs.test.ts @@ -42,7 +42,7 @@ describe('groupDocPath', () => { expect(groupDocPath({ k: 'CompressionTimeGroup' })).toBe('benchmarks/compress-bench/README.md'); // Two measurement kinds of one benchmark, so one doc. expect(groupDocPath({ k: 'CompressionSizeGroup' })).toBe('benchmarks/compress-bench/README.md'); - expect(groupDocPath({ k: 'RandomAccessGroup' })).toBe( + expect(groupDocPath({ k: 'RandomAccessGroup', storage: 'nvme' })).toBe( 'benchmarks/random-access-bench/README.md', ); }); diff --git a/web/lib/data-cache.test.ts b/web/lib/data-cache.test.ts index ea2464b..5c22286 100644 --- a/web/lib/data-cache.test.ts +++ b/web/lib/data-cache.test.ts @@ -63,9 +63,9 @@ describe('data-cache wrappers', () => { expect(BENCH_DATA_TAG).toBe('bench-data'); expect(DATA_CACHE_BACKSTOP_SECONDS).toBe(86400); expect(cacheCalls.map((call) => call.keyParts)).toContainEqual([ - 'data-cache:group-charts:v8:n100', + 'data-cache:group-charts:v9:n100', ]); - expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v8']); + expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v9']); for (const call of cacheCalls) { expect(call.options.tags).toEqual([BENCH_DATA_TAG]); expect(call.options.revalidate).toBe(DATA_CACHE_BACKSTOP_SECONDS); diff --git a/web/lib/data-cache.ts b/web/lib/data-cache.ts index 5b489db..14ebae3 100644 --- a/web/lib/data-cache.ts +++ b/web/lib/data-cache.ts @@ -46,7 +46,7 @@ const CACHE_OPTIONS = { tags: [BENCH_DATA_TAG], revalidate: DATA_CACHE_BACKSTOP_ * This value must change when a deployment cannot read the preceding shape. * It is independent of the producer-facing benchmark schema version. */ -const GROUP_PAYLOAD_CACHE_VERSION = 'v8'; +const GROUP_PAYLOAD_CACHE_VERSION = 'v9'; // The default last-100 group bundle, keyed by group slug. The slug is the cache // key (an `unstable_cache` argument), so one wrapper covers every group. A diff --git a/web/lib/descriptions.test.ts b/web/lib/descriptions.test.ts index ed4f636..2af41cf 100644 --- a/web/lib/descriptions.test.ts +++ b/web/lib/descriptions.test.ts @@ -7,6 +7,12 @@ import { groupDescription } from './descriptions'; // Pure string logic, so these run without Docker. They mirror the Rust // `descriptions.rs` unit tests verbatim (the v2 contract). describe('groupDescription', () => { + it.each(['NVMe', 'S3'])('describes random access on %s', (storage) => { + expect(groupDescription(`Random Access (${storage})`)).toBe( + `Tests performance of selecting arbitrary row indices from a file on ${storage} storage`, + ); + }); + it('returns the v2 static descriptions verbatim', () => { expect(groupDescription('Random Access')).toBe( 'Tests performance of selecting arbitrary row indices from a file on NVMe storage', diff --git a/web/lib/descriptions.ts b/web/lib/descriptions.ts index 5a507e4..d97a3af 100644 --- a/web/lib/descriptions.ts +++ b/web/lib/descriptions.ts @@ -40,7 +40,10 @@ export function groupDescription(name: string): string | null { function staticDescription(name: string): string | null { switch (name) { case 'Random Access': + case 'Random Access (NVMe)': return 'Tests performance of selecting arbitrary row indices from a file on NVMe storage'; + case 'Random Access (S3)': + return 'Tests performance of selecting arbitrary row indices from a file on S3 storage'; case 'Compression': return 'Measures write and scan throughput for Vortex, Parquet, and Lance files'; case 'Compression Size': diff --git a/web/lib/groups.test.ts b/web/lib/groups.test.ts index bbbd0c2..88c45a9 100644 --- a/web/lib/groups.test.ts +++ b/web/lib/groups.test.ts @@ -64,11 +64,11 @@ describe.skipIf(!dockerAvailable())( it('orders groups by the canonical GROUP_ORDER, with unknown groups last', async () => { const groups = await collectGroups(); - // `Random Access` is directly before Clickbench in GROUP_ORDER. + // Random-access sections precede Clickbench in GROUP_ORDER. expect(groups.map((g) => g.name)).toEqual([ 'Compression', 'Compression Size', - 'Random Access', + 'Random Access (NVMe)', 'TPC-H (NVMe) (SF=1)', 'cohere-large-10m / partitioned', ]); @@ -77,7 +77,7 @@ describe.skipIf(!dockerAvailable())( it('computes the random-access summary (geomean ratio to fastest)', async () => { const groups = await collectGroups(); const summary = expectDefined( - groups.find((g) => g.name === 'Random Access')?.summary, + groups.find((g) => g.name === 'Random Access (NVMe)')?.summary, 'random access summary', ); if (summary.type !== 'randomAccess') { @@ -210,7 +210,7 @@ describe.skipIf(!dockerAvailable())( it('attaches editorial descriptions and omits them where v2 has none', async () => { const groups = await collectGroups(); - expect(groups.find((g) => g.name === 'Random Access')?.description).toBe( + expect(groups.find((g) => g.name === 'Random Access (NVMe)')?.description).toBe( 'Tests performance of selecting arbitrary row indices from a file on NVMe storage', ); expect(groups.find((g) => g.name === 'TPC-H (NVMe) (SF=1)')?.description).toBe( @@ -234,7 +234,7 @@ describe.skipIf(!dockerAvailable())( expect(body.groups.map((g) => g.name)).toEqual([ 'Compression', 'Compression Size', - 'Random Access', + 'Random Access (NVMe)', 'TPC-H (NVMe) (SF=1)', 'cohere-large-10m / partitioned', ]); @@ -325,7 +325,7 @@ describe.skipIf(!dockerAvailable())( const pool = getPool(); await pool.query("DELETE FROM random_access_times WHERE open_mode = 'reopen'"); - const summary = await collectGroupSummary({ k: 'RandomAccessGroup' }); + const summary = await collectGroupSummary({ k: 'RandomAccessGroup', storage: 'nvme' }); if (summary === null || summary.type !== 'randomAccess') { throw new Error('expected a randomAccess summary'); } @@ -337,7 +337,10 @@ describe.skipIf(!dockerAvailable())( expect(summary.coldRankings).toEqual([]); const group = expectDefined( - await collectGroupCharts({ k: 'RandomAccessGroup' }, parseCommitWindow(null)), + await collectGroupCharts( + { k: 'RandomAccessGroup', storage: 'nvme' }, + parseCommitWindow(null), + ), 'cached-only random access group', ); expect(Object.keys(group.charts[0].series).sort()).toEqual([ @@ -463,6 +466,46 @@ describe.skipIf(!dockerAvailable())('summary math fidelity (testcontainers Postg ); } + it('splits random-access groups, summaries, and chart bundles by storage', async () => { + const sha = 'a'.repeat(40); + await insertCommit(sha, '2026-04-23T12:00:00Z'); + for (const dataset of ['taxi', 'taxi/uniform', 'taxi-s3', 'taxi-s3/uniform']) { + await insertRandomAccess(sha, dataset, 'vortex', 100); + } + + const groups = await collectGroups(); + expect(groups.map((group) => group.name)).toEqual([ + 'Random Access (NVMe)', + 'Random Access (S3)', + ]); + for (const [index, storage] of (['nvme', 's3'] as const).entries()) { + const datasets = storage === 'nvme' ? ['taxi', 'taxi/uniform'] : ['taxi-s3', 'taxi-s3/uniform']; + expect(groupKeyFromSlug(groups[index].slug)).toEqual({ k: 'RandomAccessGroup', storage }); + expect(groups[index].charts).toEqual( + datasets.map((dataset) => ({ + name: dataset, + slug: chartKeyToSlug({ k: 'RandomAccess', dataset }), + })), + ); + const bundle = await collectGroupCharts( + { k: 'RandomAccessGroup', storage }, + parseCommitWindow(null), + ); + expect(bundle?.charts.map((chart) => chart.display_name)).toEqual(datasets); + expect(bundle?.summary).toMatchObject({ + type: 'randomAccess', + hotRankings: [{ name: 'vortex', totalRuntime: 200, measured: 1, total: 1 }], + coldRankings: [], + }); + } + + await getPool().query("DELETE FROM random_access_times WHERE dataset IN ('taxi', 'taxi/uniform')"); + expect((await collectGroups()).map((group) => group.name)).toEqual(['Random Access (S3)']); + expect( + await collectGroupCharts({ k: 'RandomAccessGroup', storage: 'nvme' }, parseCommitWindow(null)), + ).toBeNull(); + }); + it('ranks random access over every chart, not the alphabetically first one', async () => { // The bug this pins: the old summary walked the group's chart links and // published the first populated chart's raw times under the group-wide @@ -484,7 +527,7 @@ describe.skipIf(!dockerAvailable())('summary math fidelity (testcontainers Postg await insertRandomAccess(sha, 'taxi', 'vortex-file-compressed', 1_000_000); const summary = expectDefined( - await collectGroupSummary({ k: 'RandomAccessGroup' }), + await collectGroupSummary({ k: 'RandomAccessGroup', storage: 'nvme' }), 'random-access summary', ); if (summary.type !== 'randomAccess') { @@ -512,7 +555,7 @@ describe.skipIf(!dockerAvailable())('summary math fidelity (testcontainers Postg await insertRandomAccess(newer, 'taxi', 'vortex-file-compressed', 1_000_000); const summary = expectDefined( - await collectGroupSummary({ k: 'RandomAccessGroup' }), + await collectGroupSummary({ k: 'RandomAccessGroup', storage: 'nvme' }), 'random-access summary', ); if (summary.type !== 'randomAccess') { @@ -536,7 +579,7 @@ describe.skipIf(!dockerAvailable())('summary math fidelity (testcontainers Postg await insertRandomAccess(sha, 'taxi', 'vortex-file-compressed', 50_000); const summary = expectDefined( - await collectGroupSummary({ k: 'RandomAccessGroup' }), + await collectGroupSummary({ k: 'RandomAccessGroup', storage: 'nvme' }), 'random-access summary', ); if (summary.type !== 'randomAccess') { diff --git a/web/lib/queries.test.ts b/web/lib/queries.test.ts index be116d6..aa72191 100644 --- a/web/lib/queries.test.ts +++ b/web/lib/queries.test.ts @@ -330,24 +330,29 @@ describe.skipIf(!dockerAvailable())('chartPayload (testcontainers Postgres)', () // exercised end-to-end only by the Docker-gated collectGroups test; this no-DB // test pins the ordering directly so it stays verifiable without testcontainers. describe('compareGroupSortKey canonical group ordering (no DB)', () => { - it('orders Random Access directly before Clickbench', () => { - const sorted = ['PolarSignals Profiling', 'Random Access', 'Clickbench', 'Compression'].sort( - compareGroupSortKey, - ); + it('orders NVMe and S3 Random Access directly before Clickbench', () => { + const sorted = [ + 'PolarSignals Profiling', + 'Random Access (S3)', + 'Random Access (NVMe)', + 'Clickbench', + 'Compression', + ].sort(compareGroupSortKey); expect(sorted).toEqual([ 'Compression', - 'Random Access', + 'Random Access (NVMe)', + 'Random Access (S3)', 'Clickbench', 'PolarSignals Profiling', ]); - expect(sorted.indexOf('Random Access') + 1).toBe(sorted.indexOf('Clickbench')); + expect(sorted.indexOf('Random Access (S3)') + 1).toBe(sorted.indexOf('Clickbench')); }); it('sorts listed groups before unknown groups, unknowns alphabetically last', () => { - const sorted = ['cohere-large-10m / partitioned', 'Random Access', 'Compression'].sort( + const sorted = ['cohere-large-10m / partitioned', 'Random Access (NVMe)', 'Compression'].sort( compareGroupSortKey, ); - expect(sorted).toEqual(['Compression', 'Random Access', 'cohere-large-10m / partitioned']); + expect(sorted).toEqual(['Compression', 'Random Access (NVMe)', 'cohere-large-10m / partitioned']); }); it('places the appian group directly after both fineweb groups', () => { diff --git a/web/lib/queries.ts b/web/lib/queries.ts index dd42fee..6ae1741 100644 --- a/web/lib/queries.ts +++ b/web/lib/queries.ts @@ -31,6 +31,7 @@ */ import { getPool } from './db'; +import { randomAccessStorage } from './random-access'; import { groupDescription } from './descriptions'; import type { FilterUniverse } from './chart-format'; import { compareCodeUnits, FAMILIES, type GroupKind } from './families'; @@ -685,7 +686,8 @@ export interface GroupChartsResponse { const GROUP_ORDER: readonly string[] = [ 'Compression', 'Compression Size', - 'Random Access', + 'Random Access (NVMe)', + 'Random Access (S3)', 'Clickbench', 'TPC-H (NVMe) (SF=1)', 'TPC-H (S3) (SF=1)', @@ -1001,7 +1003,7 @@ async function collectCompressionSizeGroup(): Promise { ]; } -/** The single `Random Access` group, or `[]` if the fact table is empty. */ +/** One random-access group per storage backend, omitting empty sections. */ async function collectRandomAccessGroup(): Promise { const text = ` SELECT DISTINCT dataset @@ -1009,14 +1011,23 @@ async function collectRandomAccessGroup(): Promise { ORDER BY dataset `; const rows = (await getPool().query<{ dataset: string }>(text)).rows; - if (rows.length === 0) { - return []; + const groups: Group[] = []; + for (const storage of ['nvme', 's3'] as const) { + const charts: ChartLink[] = rows + .filter((row) => randomAccessStorage(row.dataset) === storage) + .map((row) => ({ + name: row.dataset, + slug: chartKeyToSlug({ k: 'RandomAccess', dataset: row.dataset }), + })); + if (charts.length > 0) { + groups.push({ + name: `Random Access (${storage === 'nvme' ? 'NVMe' : 'S3'})`, + slug: groupKeyToSlug({ k: 'RandomAccessGroup', storage }), + charts, + }); + } } - const charts: ChartLink[] = rows.map((row) => ({ - name: row.dataset, - slug: chartKeyToSlug({ k: 'RandomAccess', dataset: row.dataset }), - })); - return [{ name: 'Random Access', slug: groupKeyToSlug({ k: 'RandomAccessGroup' }), charts }]; + return groups; } type VectorSearchGroupRow = { dataset: string; layout: string; threshold: number }; diff --git a/web/lib/random-access.test.ts b/web/lib/random-access.test.ts new file mode 100644 index 0000000..efb140e --- /dev/null +++ b/web/lib/random-access.test.ts @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +import { describe, expect, it } from 'vitest'; +import { randomAccessStorage } from './random-access'; + +// Matches push_v3_random_access_record in the monorepo random-access producer. +describe('randomAccessStorage', () => { + it.each([ + ['taxi', 'nvme'], + ['taxi/correlated', 'nvme'], + ['feature-vectors/uniform', 'nvme'], + ['taxi-s3', 's3'], + ['taxi-s3/correlated', 's3'], + ['feature-vectors-s3/uniform', 's3'], + ['taxi/pattern-s3', 'nvme'], + ['taxi-s3-copy/uniform', 'nvme'], + ] as const)('assigns %s to %s', (dataset, storage) => { + expect(randomAccessStorage(dataset)).toBe(storage); + }); +}); diff --git a/web/lib/random-access.ts b/web/lib/random-access.ts new file mode 100644 index 0000000..20c7526 --- /dev/null +++ b/web/lib/random-access.ts @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +export type RandomAccessStorage = 'nvme' | 's3'; + +/** + * The producer encodes S3 storage as `{dataset}-s3/{pattern}` (or legacy + * `taxi-s3`), since random_access_times has no separate storage dimension. + */ +export function randomAccessStorage(dataset: string): RandomAccessStorage { + return dataset.split('/')[0].endsWith('-s3') ? 's3' : 'nvme'; +} diff --git a/web/lib/slug.test.ts b/web/lib/slug.test.ts index b8aadde..6710a4c 100644 --- a/web/lib/slug.test.ts +++ b/web/lib/slug.test.ts @@ -31,7 +31,8 @@ const GROUP_KEYS: GroupKey[] = [ { k: 'QueryGroup', dataset: 'tpch', dataset_variant: null, scale_factor: '1', storage: 'nvme' }, { k: 'CompressionTimeGroup' }, { k: 'CompressionSizeGroup' }, - { k: 'RandomAccessGroup' }, + { k: 'RandomAccessGroup', storage: 'nvme' }, + { k: 'RandomAccessGroup', storage: 's3' }, { k: 'VectorSearchGroup', dataset: 'cohere', layout: 'partitioned' }, ]; @@ -83,6 +84,17 @@ describe('group slug round-trip', () => { expect(chartPrefix).not.toBe(groupPrefix); }); + it('resolves legacy random-access group slugs to NVMe', () => { + const legacy = 'rag.' + Buffer.from('{"k":"RandomAccessGroup"}').toString('base64url'); + expect(groupKeyFromSlug(legacy)).toEqual({ k: 'RandomAccessGroup', storage: 'nvme' }); + }); + + it.each(['disk', '', null, 1])('rejects invalid random-access storage %s', (storage) => { + const slug = + 'rag.' + Buffer.from(JSON.stringify({ k: 'RandomAccessGroup', storage })).toString('base64url'); + expect(() => groupKeyFromSlug(slug)).toThrow(); + }); + it('rejects a malformed group slug', () => { expect(() => groupKeyFromSlug('not-a-slug')).toThrow(); expect(() => groupKeyFromSlug('qmg.****')).toThrow(); diff --git a/web/lib/slug.ts b/web/lib/slug.ts index dadf062..ea1178f 100644 --- a/web/lib/slug.ts +++ b/web/lib/slug.ts @@ -16,6 +16,7 @@ * a slug back gives a strongly-typed [`ChartKey`] or [`GroupKey`]. */ +import type { RandomAccessStorage } from './random-access'; import { familyForChartKind, familyForGroupKind, type ChartKind, type GroupKind } from './families'; /** Strongly-typed chart key parsed from a slug. Discriminated on `k`. */ @@ -44,7 +45,7 @@ export type GroupKey = } | { k: 'CompressionTimeGroup' } | { k: 'CompressionSizeGroup' } - | { k: 'RandomAccessGroup' } + | { k: 'RandomAccessGroup'; storage: RandomAccessStorage } | { k: 'VectorSearchGroup'; dataset: string; layout: string }; const CHART_KINDS: readonly ChartKind[] = [ @@ -234,9 +235,11 @@ export function groupKeyToSlug(key: GroupKey): string { case 'VectorSearchGroup': ordered = { k: key.k, dataset: key.dataset, layout: key.layout }; break; + case 'RandomAccessGroup': + ordered = { k: key.k, storage: key.storage }; + break; case 'CompressionTimeGroup': case 'CompressionSizeGroup': - case 'RandomAccessGroup': ordered = { k: key.k }; break; } @@ -265,9 +268,16 @@ export function groupKeyFromSlug(slug: string): GroupKey { }; case 'VectorSearchGroup': return { k: kind, dataset: reqString(obj, 'dataset'), layout: reqString(obj, 'layout') }; + case 'RandomAccessGroup': { + // Group links from before the storage split resolve to the NVMe section. + const storage = obj.storage === undefined ? 'nvme' : reqString(obj, 'storage'); + if (storage !== 'nvme' && storage !== 's3') { + throw new Error('random-access group storage must be nvme or s3'); + } + return { k: kind, storage }; + } case 'CompressionTimeGroup': case 'CompressionSizeGroup': - case 'RandomAccessGroup': return { k: kind }; } } diff --git a/web/lib/summary.test.ts b/web/lib/summary.test.ts index f563482..8d38809 100644 --- a/web/lib/summary.test.ts +++ b/web/lib/summary.test.ts @@ -198,6 +198,38 @@ describe('timing summaries (shared ranking model)', () => { query.mockReset(); }); + it.each(['cached', 'reopen'] as const)( + 'ranks NVMe and S3 independently for %s access', + async (openMode) => { + query.mockResolvedValue({ + rows: [ + { bucket: 'taxi', series: 'vortex', value: 100, open_mode: openMode }, + { bucket: 'taxi', series: 'lance', value: 200, open_mode: openMode }, + { bucket: 'taxi', series: 'arrow-ipc', value: 300, open_mode: openMode }, + { bucket: 'taxi-s3', series: 'vortex', value: 20_000, open_mode: openMode }, + { bucket: 'taxi-s3', series: 'lance', value: 10_000, open_mode: openMode }, + { bucket: 'taxi-s3/uniform', series: 'vortex', value: 40_000, open_mode: openMode }, + { bucket: 'taxi-s3/uniform', series: 'lance', value: 20_000, open_mode: openMode }, + ], + }); + const nvme = await collectGroupSummary({ k: 'RandomAccessGroup', storage: 'nvme' }); + const s3 = await collectGroupSummary({ k: 'RandomAccessGroup', storage: 's3' }); + if (nvme?.type !== 'randomAccess' || s3?.type !== 'randomAccess') { + throw new Error('expected randomAccess summaries'); + } + const panel = openMode === 'cached' ? 'hotRankings' : 'coldRankings'; + expect(nvme[panel].map((r) => [r.name, r.totalRuntime, r.measured, r.total])).toEqual([ + ['vortex', 100, 1, 1], + ['lance', 200, 1, 1], + ['arrow-ipc', 300, 1, 1], + ]); + expect(s3[panel].map((r) => [r.name, r.totalRuntime, r.measured, r.total])).toEqual([ + ['lance', 30_000, 1, 1], + ['vortex', 60_000, 1, 1], + ]); + }, + ); + it('sums random-access charts by dataset before ranking', async () => { // The regression: the old summary published one chart's raw times under the // group-wide title. `lance` wins `feature-vectors/correlated` outright and @@ -213,7 +245,7 @@ describe('timing summaries (shared ranking model)', () => { ], }); - const summary = await collectGroupSummary({ k: 'RandomAccessGroup' }); + const summary = await collectGroupSummary({ k: 'RandomAccessGroup', storage: 'nvme' }); if (summary === null || summary.type !== 'randomAccess') { throw new Error('expected a randomAccess summary'); } @@ -242,7 +274,7 @@ describe('timing summaries (shared ranking model)', () => { ], }); - const summary = await collectGroupSummary({ k: 'RandomAccessGroup' }); + const summary = await collectGroupSummary({ k: 'RandomAccessGroup', storage: 'nvme' }); if (summary === null || summary.type !== 'randomAccess') { throw new Error('expected a randomAccess summary'); } @@ -266,7 +298,7 @@ describe('timing summaries (shared ranking model)', () => { it('reads each random-access format at its own newest run', async () => { query.mockResolvedValue({ rows: [] }); - await collectGroupSummary({ k: 'RandomAccessGroup' }); + await collectGroupSummary({ k: 'RandomAccessGroup', storage: 'nvme' }); const [text] = query.mock.calls[0] as [string, unknown[] | undefined]; // Per-series freshness, not one global latest commit: a format that skipped // the newest commit stays on the card at its own last run. @@ -289,7 +321,7 @@ describe('timing summaries (shared ranking model)', () => { ], }); - const summary = await collectGroupSummary({ k: 'RandomAccessGroup' }); + const summary = await collectGroupSummary({ k: 'RandomAccessGroup', storage: 'nvme' }); if (summary === null || summary.type !== 'randomAccess') { throw new Error('expected a randomAccess summary'); } @@ -314,7 +346,7 @@ describe('timing summaries (shared ranking model)', () => { ], }); - const summary = await collectGroupSummary({ k: 'RandomAccessGroup' }); + const summary = await collectGroupSummary({ k: 'RandomAccessGroup', storage: 'nvme' }); if (summary === null || summary.type !== 'randomAccess') { throw new Error('expected a randomAccess summary'); } @@ -336,7 +368,7 @@ describe('timing summaries (shared ranking model)', () => { ], }); - const summary = await collectGroupSummary({ k: 'RandomAccessGroup' }); + const summary = await collectGroupSummary({ k: 'RandomAccessGroup', storage: 'nvme' }); if (summary === null || summary.type !== 'randomAccess') { throw new Error('expected a randomAccess summary'); } @@ -360,7 +392,7 @@ describe('timing summaries (shared ranking model)', () => { ], }); - const summary = await collectGroupSummary({ k: 'RandomAccessGroup' }); + const summary = await collectGroupSummary({ k: 'RandomAccessGroup', storage: 'nvme' }); if (summary === null || summary.type !== 'randomAccess') { throw new Error('expected a randomAccess summary'); } @@ -430,7 +462,7 @@ describe('timing summaries (shared ranking model)', () => { }, { k: 'CompressionTimeGroup' }, { k: 'CompressionSizeGroup' }, - { k: 'RandomAccessGroup' }, + { k: 'RandomAccessGroup', storage: 'nvme' }, { k: 'VectorSearchGroup', dataset: 'd', layout: 'l' }, ]; expect(keys.map((key) => key.k).sort()).toEqual([...GROUP_KINDS].sort()); diff --git a/web/lib/summary.ts b/web/lib/summary.ts index 6f53aca..453bbed 100644 --- a/web/lib/summary.ts +++ b/web/lib/summary.ts @@ -37,6 +37,7 @@ */ import { getPool } from './db'; +import { randomAccessStorage, type RandomAccessStorage } from './random-access'; import { compareCodeUnits } from './families'; import type { GroupKey } from './slug'; @@ -180,7 +181,7 @@ export function collectGroupSummary(key: GroupKey): Promise { case 'CompressionSizeGroup': return collectCompressionSizeSummary(); case 'RandomAccessGroup': - return collectRandomAccessSummary(); + return collectRandomAccessSummary(key.storage); case 'VectorSearchGroup': return collectVectorSearchSummary(key.dataset, key.layout); } @@ -392,7 +393,7 @@ function groupRandomAccessSamples( } /** - * The random-access rollup, over one summed bucket per dataset and open mode. + * The random-access rollup for one storage backend, over dataset and open-mode buckets. * * Two things this deliberately does NOT do, both of which the previous * implementation did: @@ -418,7 +419,7 @@ function groupRandomAccessSamples( * statement exceed the server-side statement timeout in production. The * table is small, so the per-series `DISTINCT ON` descent is cheap. */ -async function collectRandomAccessSummary(): Promise { +async function collectRandomAccessSummary(storage: RandomAccessStorage): Promise { const text = ` SELECT DISTINCT ON (r.dataset, r.format, r.open_mode) r.dataset AS bucket, @@ -441,7 +442,10 @@ async function collectRandomAccessSummary(): Promise { }>(text) ).rows; const rankingsFor = (openMode: 'cached' | 'reopen') => { - const modeRows = rows.filter((row) => (row.open_mode ?? 'cached') === openMode); + const modeRows = rows.filter( + (row) => + randomAccessStorage(row.bucket) === storage && (row.open_mode ?? 'cached') === openMode, + ); const grouped = groupRandomAccessSamples(modeRows); const knownSeries = [...new Set(modeRows.map((row) => row.series))]; return rankSeries(grouped, compareCodeUnits, 0, 2, knownSeries).map((ranking) => ({ From 5676087fa1dd02dab33c985d0f12c36543c94306 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 9 Oct 2026 14:28:32 +0100 Subject: [PATCH 2/2] Format random-access storage regression tests Signed-off-by: Joe Isaacs --- web/lib/groups.test.ts | 12 +++++++++--- web/lib/queries.test.ts | 6 +++++- web/lib/slug.test.ts | 3 ++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/web/lib/groups.test.ts b/web/lib/groups.test.ts index 88c45a9..c9979bd 100644 --- a/web/lib/groups.test.ts +++ b/web/lib/groups.test.ts @@ -479,7 +479,8 @@ describe.skipIf(!dockerAvailable())('summary math fidelity (testcontainers Postg 'Random Access (S3)', ]); for (const [index, storage] of (['nvme', 's3'] as const).entries()) { - const datasets = storage === 'nvme' ? ['taxi', 'taxi/uniform'] : ['taxi-s3', 'taxi-s3/uniform']; + const datasets = + storage === 'nvme' ? ['taxi', 'taxi/uniform'] : ['taxi-s3', 'taxi-s3/uniform']; expect(groupKeyFromSlug(groups[index].slug)).toEqual({ k: 'RandomAccessGroup', storage }); expect(groups[index].charts).toEqual( datasets.map((dataset) => ({ @@ -499,10 +500,15 @@ describe.skipIf(!dockerAvailable())('summary math fidelity (testcontainers Postg }); } - await getPool().query("DELETE FROM random_access_times WHERE dataset IN ('taxi', 'taxi/uniform')"); + await getPool().query( + "DELETE FROM random_access_times WHERE dataset IN ('taxi', 'taxi/uniform')", + ); expect((await collectGroups()).map((group) => group.name)).toEqual(['Random Access (S3)']); expect( - await collectGroupCharts({ k: 'RandomAccessGroup', storage: 'nvme' }, parseCommitWindow(null)), + await collectGroupCharts( + { k: 'RandomAccessGroup', storage: 'nvme' }, + parseCommitWindow(null), + ), ).toBeNull(); }); diff --git a/web/lib/queries.test.ts b/web/lib/queries.test.ts index aa72191..f9280e2 100644 --- a/web/lib/queries.test.ts +++ b/web/lib/queries.test.ts @@ -352,7 +352,11 @@ describe('compareGroupSortKey canonical group ordering (no DB)', () => { const sorted = ['cohere-large-10m / partitioned', 'Random Access (NVMe)', 'Compression'].sort( compareGroupSortKey, ); - expect(sorted).toEqual(['Compression', 'Random Access (NVMe)', 'cohere-large-10m / partitioned']); + expect(sorted).toEqual([ + 'Compression', + 'Random Access (NVMe)', + 'cohere-large-10m / partitioned', + ]); }); it('places the appian group directly after both fineweb groups', () => { diff --git a/web/lib/slug.test.ts b/web/lib/slug.test.ts index 6710a4c..b944591 100644 --- a/web/lib/slug.test.ts +++ b/web/lib/slug.test.ts @@ -91,7 +91,8 @@ describe('group slug round-trip', () => { it.each(['disk', '', null, 1])('rejects invalid random-access storage %s', (storage) => { const slug = - 'rag.' + Buffer.from(JSON.stringify({ k: 'RandomAccessGroup', storage })).toString('base64url'); + 'rag.' + + Buffer.from(JSON.stringify({ k: 'RandomAccessGroup', storage })).toString('base64url'); expect(() => groupKeyFromSlug(slug)).toThrow(); });