From 921a20f7067f5bba85c5c216733ec7efa1573da9 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 3 Sep 2026 00:49:41 -0700 Subject: [PATCH 1/2] Add tests for seeded catalog offering priorities (CL-7294) Anthropic currently seeds every model at the same provider-level priority, so localeCompare picks claude-fable-5 over the declared default. A 409 on an existing offering is skipped forever, so a re-seed cannot repair that. These tests pin distinct declared priorities and a GET/PATCH reconcile on conflict. --- packages/seeding/test/seed.test.ts | 182 +++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) diff --git a/packages/seeding/test/seed.test.ts b/packages/seeding/test/seed.test.ts index 31a945f61..77e9c4193 100644 --- a/packages/seeding/test/seed.test.ts +++ b/packages/seeding/test/seed.test.ts @@ -1560,6 +1560,188 @@ describe("seedCatalog", () => { ); }); + test.failing( + "fresh run gives the declared Anthropic default the lowest distinct priority", + async () => { + const { log } = collector(); + const modelNamesById = new Map(); + const offeringPosts: { modelId: string; priority: number }[] = []; + const handler: FakeHandler = (method, path, body) => { + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) + return { status: 201, data: providerRow("prv_1", "anthropic") }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/credentials` + ) + return { + status: 201, + data: credentialRow("cre_1", "prv_1", "anthropic-default"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/models` + ) { + const canonicalName = (body as { canonicalName: string }) + .canonicalName; + const modelId = `mdl_${modelNamesById.size + 1}`; + modelNamesById.set(modelId, canonicalName); + return { + status: 201, + data: catalogModelRow(modelId, canonicalName), + }; + } + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/providers` + ) + return { + status: 201, + data: catalogProviderRow("cpv_1", "anthropic", "cre_1"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + const offering = body as { modelId: string; priority: number }; + offeringPosts.push(offering); + return { + status: 201, + data: catalogOfferingRow( + `off_${offeringPosts.length}`, + offering.modelId, + "cpv_1", + ), + }; + } + return undefined; + }; + + await seedCatalog({ + api: fakeAPI(handler), + cookies: [], + tenantId: TENANT_ID, + apiKey: "sk-test", + log, + }); + + const priorities = offeringPosts.map((offering) => offering.priority); + const sonnet = offeringPosts.find( + (offering) => + modelNamesById.get(offering.modelId) === "claude-sonnet-5", + ); + expect(new Set(priorities).size).toBe(offeringPosts.length); + expect(sonnet?.priority).toBe(Math.min(...priorities)); + }, + ); + + test.failing( + "re-run updates a legacy offering to its computed priority", + async () => { + const { log } = collector(); + const patchedOfferings: { id: string; priority: number }[] = []; + const handler: FakeHandler = (method, path, body) => { + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) + return { status: 201, data: providerRow("prv_1", "anthropic") }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/credentials` + ) + return { + status: 201, + data: credentialRow("cre_1", "prv_1", "anthropic-default"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/models` + ) { + const canonicalName = (body as { canonicalName: string }) + .canonicalName; + const modelId = + canonicalName === "claude-opus-5" ? "mdl_legacy" : "mdl_new"; + return { status: 201, data: catalogModelRow(modelId, canonicalName) }; + } + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/providers` + ) + return { + status: 201, + data: catalogProviderRow("cpv_1", "anthropic", "cre_1"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + const offering = body as { modelId: string }; + if (offering.modelId === "mdl_legacy") + return { status: 409, data: { error: "already exists" } }; + return { + status: 201, + data: catalogOfferingRow( + `off_${offering.modelId}`, + offering.modelId, + "cpv_1", + ), + }; + } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) + return { + status: 200, + data: { + data: [ + catalogOfferingRow( + "off_other_provider", + "mdl_legacy", + "cpv_other", + ), + ], + nextCursor: "second-page", + }, + }; + if ( + method === "GET" && + path === + `/api/tenants/${TENANT_ID}/catalog/offerings?cursor=second-page` + ) + return { + status: 200, + data: { + data: [catalogOfferingRow("off_legacy", "mdl_legacy", "cpv_1")], + nextCursor: null, + }, + }; + if ( + method === "PATCH" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings/off_legacy` + ) { + const patch = body as { priority: number }; + patchedOfferings.push({ id: "off_legacy", priority: patch.priority }); + return { + status: 200, + data: { + ...catalogOfferingRow("off_legacy", "mdl_legacy", "cpv_1"), + priority: patch.priority, + }, + }; + } + return undefined; + }; + + await seedCatalog({ + api: fakeAPI(handler), + cookies: [], + tenantId: TENANT_ID, + apiKey: "sk-test", + log, + }); + + expect(patchedOfferings).toEqual([{ id: "off_legacy", priority: 1 }]); + }, + ); + test("an Ollama offering's quirks carry that model's real context-window ceiling, not the built-in 4096 default", async () => { const { log } = collector(); const offeringBodies: Record[] = []; From 757d38a58e69debebc440fcfbf59ff6e722b2c5f Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 3 Sep 2026 00:53:55 -0700 Subject: [PATCH 2/2] Seed catalog offerings in declared priority order (CL-7294) Flatten CATALOG_SEEDS into one priority sequence so Anthropic claude-sonnet-5 is priority 0 instead of tying with every other Anthropic model. A 409 now lists the existing offering and PATCHes priority when it has drifted. --- .../test/complete-credential.test.ts | 62 ++- .../test/huggingface-connect-routes.test.ts | 3 + .../test/openrouter-connect-routes.test.ts | 3 + packages/seeding/src/seed.ts | 67 +++- packages/seeding/test/seed.test.ts | 374 +++++++++--------- 5 files changed, 311 insertions(+), 198 deletions(-) diff --git a/packages/onboarding/test/complete-credential.test.ts b/packages/onboarding/test/complete-credential.test.ts index 87349592c..02034f612 100644 --- a/packages/onboarding/test/complete-credential.test.ts +++ b/packages/onboarding/test/complete-credential.test.ts @@ -753,6 +753,17 @@ describe("completeCredentialSetup", () => { cookies: [], }; } + if ( + method === "GET" && + (path === `/api/tenants/${TENANT_ID}/catalog/offerings` || + path.startsWith(`/api/tenants/${TENANT_ID}/catalog/offerings?`)) + ) { + return { + status: 200, + data: { data: [], nextCursor: null }, + cookies: [], + }; + } throw new Error(`unexpected call: ${method} ${path}`); }; @@ -1004,7 +1015,12 @@ describe("completeCredentialSetup", () => { const deployments: { definitionAssetId: string; id: string }[] = []; const catalogModels: Row[] = []; const catalogProviders: Row[] = []; - const catalogOfferings: { modelId: string; providerId: string }[] = []; + const catalogOfferings: { + id: string; + modelId: string; + providerId: string; + priority: number; + }[] = []; const providers: Row[] = []; const credentials: Row[] = []; let assetCreatePosts = 0; @@ -1407,21 +1423,31 @@ describe("completeCredentialSetup", () => { method === "POST" && path === `/api/tenants/${TENANT_ID}/catalog/offerings` ) { - const b = body as { modelId: string; providerId: string }; + const b = body as { + modelId: string; + providerId: string; + priority: number; + }; const existing = catalogOfferings.find( (o) => o.modelId === b.modelId && o.providerId === b.providerId, ); if (existing) return { status: 409, data: {}, cookies: [] }; catalogOfferingCreatePosts += 1; - catalogOfferings.push({ modelId: b.modelId, providerId: b.providerId }); + const id = `off_${catalogOfferings.length + 1}`; + catalogOfferings.push({ + id, + modelId: b.modelId, + providerId: b.providerId, + priority: b.priority, + }); return { status: 201, data: { - id: `off_${catalogOfferings.length}`, + id, tenantId: TENANT_ID, modelId: b.modelId, providerId: b.providerId, - priority: 0, + priority: b.priority, deploymentTags: [], capabilities: [], quirks: null, @@ -1432,6 +1458,32 @@ describe("completeCredentialSetup", () => { cookies: [], }; } + if ( + method === "GET" && + (path === `/api/tenants/${TENANT_ID}/catalog/offerings` || + path.startsWith(`/api/tenants/${TENANT_ID}/catalog/offerings?`)) + ) { + return { + status: 200, + data: { + data: catalogOfferings.map((o) => ({ + id: o.id, + tenantId: TENANT_ID, + modelId: o.modelId, + providerId: o.providerId, + priority: o.priority, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: TIMESTAMP, + updatedAt: TIMESTAMP, + })), + nextCursor: null, + }, + cookies: [], + }; + } throw new Error(`unexpected call: ${method} ${path}`); }; diff --git a/packages/onboarding/test/huggingface-connect-routes.test.ts b/packages/onboarding/test/huggingface-connect-routes.test.ts index 19905bee8..f78971cc5 100644 --- a/packages/onboarding/test/huggingface-connect-routes.test.ts +++ b/packages/onboarding/test/huggingface-connect-routes.test.ts @@ -156,6 +156,9 @@ function mockHub() { 201, ), ); + hub.get("/api/tenants/ten_1/catalog/offerings", (c) => + c.json({ data: [], nextCursor: null }), + ); return hub; } diff --git a/packages/onboarding/test/openrouter-connect-routes.test.ts b/packages/onboarding/test/openrouter-connect-routes.test.ts index a18304407..27454ab4c 100644 --- a/packages/onboarding/test/openrouter-connect-routes.test.ts +++ b/packages/onboarding/test/openrouter-connect-routes.test.ts @@ -154,6 +154,9 @@ function mockHub() { 201, ), ); + hub.get("/api/tenants/ten_1/catalog/offerings", (c) => + c.json({ data: [], nextCursor: null }), + ); return hub; } diff --git a/packages/seeding/src/seed.ts b/packages/seeding/src/seed.ts index ac636d18d..bc76c435e 100644 --- a/packages/seeding/src/seed.ts +++ b/packages/seeding/src/seed.ts @@ -1343,7 +1343,52 @@ async function ensureCatalogOffering( return; } if (created.status === 409) { - log("catalog offering already exists (skipped)"); + let cursor: string | null = null; + let existing: typeof ModelOfferingResponse.infer | undefined; + do { + const listed = await api( + "GET", + `/api/tenants/${args.tenantId}/catalog/offerings${cursor === null ? "" : `?cursor=${encodeURIComponent(cursor)}`}`, + undefined, + cookies, + ); + const page = parseAs( + paginatedSchema(ModelOfferingResponse), + listed.data, + "catalog offerings response", + ); + existing = page.data.find( + (offering) => + offering.modelId === args.modelId && + offering.providerId === args.providerId, + ); + cursor = page.nextCursor; + } while (existing === undefined && cursor !== null); + if (!existing) { + throw new HubApiError( + "catalog offering reported a conflict but is not listable on the bench", + "check the hub logs for the underlying failure, then re-run: workbench seed", + ); + } + if (existing.priority === args.priority) { + log("catalog offering already exists (skipped)"); + return; + } + + const updated = await api( + "PATCH", + `/api/tenants/${args.tenantId}/catalog/offerings/${existing.id}`, + { priority: args.priority }, + cookies, + ); + if (updated.status !== 200) { + throw new HubApiError( + `the hub rejected updating the catalog offering priority with status ${updated.status}: ${JSON.stringify(updated.data)}`, + "check the hub logs for the underlying failure, then re-run: workbench seed", + ); + } + parseAs(ModelOfferingResponse, updated.data, "catalog offering response"); + log("updated catalog offering priority"); return; } throw new HubApiError( @@ -1570,14 +1615,14 @@ export async function seedCatalog( }, log, ); - // Priority = the provider's CATALOG_SEEDS declaration index (anthropic - // first), so when several connected providers serve the same model the - // fallback order is deterministic instead of an all-zeroes tie broken - // by insertion accident. - const offeringPriority = Math.max( - 0, - Object.keys(CATALOG_SEEDS).indexOf(provider), - ); + // Flatten the curated provider/model declaration order into one priority + // sequence. Provider order still controls cross-provider fallback, while + // model order makes each provider's declared default the first choice. + let offeringPriorityOffset = 0; + for (const [seedProvider, providerSeed] of Object.entries(CATALOG_SEEDS)) { + if (seedProvider === provider) break; + offeringPriorityOffset += providerSeed.models.length; + } // What each deployment can do, resolved from the pinned catalog's probe // results. Until this, every seeded offering stored an empty capability // list, so no capability filter — this repo's concept resolution or the @@ -1589,7 +1634,7 @@ export async function seedCatalog( canonicalName: string; capabilities: readonly string[]; }[] = []; - for (const model of seededModels) { + for (const [modelIndex, model] of seededModels.entries()) { // Ollama's dynamic entries already carry their own live-probed // capabilities (`fetchOllamaModelCatalog`, CL-6366) — narrowed against // the real `Capability` enum here, the trust boundary, rather than @@ -1631,7 +1676,7 @@ export async function seedCatalog( tenantId, modelId: model.id, providerId: catalogProviderId, - priority: offeringPriority, + priority: offeringPriorityOffset + modelIndex, capabilities, ...(quirks !== undefined ? { quirks } : {}), }, diff --git a/packages/seeding/test/seed.test.ts b/packages/seeding/test/seed.test.ts index 77e9c4193..3bb9544c2 100644 --- a/packages/seeding/test/seed.test.ts +++ b/packages/seeding/test/seed.test.ts @@ -1505,7 +1505,7 @@ describe("seedCatalog", () => { capabilities: string[]; }; expect(offeringBody.providerId).toBe("cpv_1"); - expect(offeringBody.priority).toBe(0); + expect(offeringBody.priority).toBe(offeringPosts.length); expect(offeringBody.capabilities.length).toBeGreaterThan(0); expect(offeringBody.capabilities).toContain("plain-text"); expect(offeringBody.capabilities).toContain( @@ -1560,187 +1560,172 @@ describe("seedCatalog", () => { ); }); - test.failing( - "fresh run gives the declared Anthropic default the lowest distinct priority", - async () => { - const { log } = collector(); - const modelNamesById = new Map(); - const offeringPosts: { modelId: string; priority: number }[] = []; - const handler: FakeHandler = (method, path, body) => { - if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) - return { status: 201, data: providerRow("prv_1", "anthropic") }; - if ( - method === "POST" && - path === `/api/tenants/${TENANT_ID}/credentials` - ) - return { - status: 201, - data: credentialRow("cre_1", "prv_1", "anthropic-default"), - }; - if ( - method === "POST" && - path === `/api/tenants/${TENANT_ID}/catalog/models` - ) { - const canonicalName = (body as { canonicalName: string }) - .canonicalName; - const modelId = `mdl_${modelNamesById.size + 1}`; - modelNamesById.set(modelId, canonicalName); - return { - status: 201, - data: catalogModelRow(modelId, canonicalName), - }; - } - if ( - method === "POST" && - path === `/api/tenants/${TENANT_ID}/catalog/providers` - ) - return { - status: 201, - data: catalogProviderRow("cpv_1", "anthropic", "cre_1"), - }; - if ( - method === "POST" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` - ) { - const offering = body as { modelId: string; priority: number }; - offeringPosts.push(offering); - return { - status: 201, - data: catalogOfferingRow( - `off_${offeringPosts.length}`, - offering.modelId, - "cpv_1", - ), - }; - } - return undefined; - }; + test("fresh run gives the declared Anthropic default the lowest distinct priority", async () => { + const { log } = collector(); + const modelNamesById = new Map(); + const offeringPosts: { modelId: string; priority: number }[] = []; + const handler: FakeHandler = (method, path, body) => { + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) + return { status: 201, data: providerRow("prv_1", "anthropic") }; + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/credentials`) + return { + status: 201, + data: credentialRow("cre_1", "prv_1", "anthropic-default"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/models` + ) { + const canonicalName = (body as { canonicalName: string }).canonicalName; + const modelId = `mdl_${modelNamesById.size + 1}`; + modelNamesById.set(modelId, canonicalName); + return { + status: 201, + data: catalogModelRow(modelId, canonicalName), + }; + } + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/providers` + ) + return { + status: 201, + data: catalogProviderRow("cpv_1", "anthropic", "cre_1"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + const offering = body as { modelId: string; priority: number }; + offeringPosts.push(offering); + return { + status: 201, + data: catalogOfferingRow( + `off_${offeringPosts.length}`, + offering.modelId, + "cpv_1", + ), + }; + } + return undefined; + }; - await seedCatalog({ - api: fakeAPI(handler), - cookies: [], - tenantId: TENANT_ID, - apiKey: "sk-test", - log, - }); - - const priorities = offeringPosts.map((offering) => offering.priority); - const sonnet = offeringPosts.find( - (offering) => - modelNamesById.get(offering.modelId) === "claude-sonnet-5", - ); - expect(new Set(priorities).size).toBe(offeringPosts.length); - expect(sonnet?.priority).toBe(Math.min(...priorities)); - }, - ); - - test.failing( - "re-run updates a legacy offering to its computed priority", - async () => { - const { log } = collector(); - const patchedOfferings: { id: string; priority: number }[] = []; - const handler: FakeHandler = (method, path, body) => { - if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) - return { status: 201, data: providerRow("prv_1", "anthropic") }; - if ( - method === "POST" && - path === `/api/tenants/${TENANT_ID}/credentials` - ) - return { - status: 201, - data: credentialRow("cre_1", "prv_1", "anthropic-default"), - }; - if ( - method === "POST" && - path === `/api/tenants/${TENANT_ID}/catalog/models` - ) { - const canonicalName = (body as { canonicalName: string }) - .canonicalName; - const modelId = - canonicalName === "claude-opus-5" ? "mdl_legacy" : "mdl_new"; - return { status: 201, data: catalogModelRow(modelId, canonicalName) }; - } - if ( - method === "POST" && - path === `/api/tenants/${TENANT_ID}/catalog/providers` - ) - return { - status: 201, - data: catalogProviderRow("cpv_1", "anthropic", "cre_1"), - }; - if ( - method === "POST" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` - ) { - const offering = body as { modelId: string }; - if (offering.modelId === "mdl_legacy") - return { status: 409, data: { error: "already exists" } }; - return { - status: 201, - data: catalogOfferingRow( - `off_${offering.modelId}`, - offering.modelId, - "cpv_1", - ), - }; - } - if ( - method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` - ) - return { - status: 200, - data: { - data: [ - catalogOfferingRow( - "off_other_provider", - "mdl_legacy", - "cpv_other", - ), - ], - nextCursor: "second-page", - }, - }; - if ( - method === "GET" && - path === - `/api/tenants/${TENANT_ID}/catalog/offerings?cursor=second-page` - ) - return { - status: 200, - data: { - data: [catalogOfferingRow("off_legacy", "mdl_legacy", "cpv_1")], - nextCursor: null, - }, - }; - if ( - method === "PATCH" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings/off_legacy` - ) { - const patch = body as { priority: number }; - patchedOfferings.push({ id: "off_legacy", priority: patch.priority }); - return { - status: 200, - data: { - ...catalogOfferingRow("off_legacy", "mdl_legacy", "cpv_1"), - priority: patch.priority, - }, - }; - } - return undefined; - }; + await seedCatalog({ + api: fakeAPI(handler), + cookies: [], + tenantId: TENANT_ID, + apiKey: "sk-test", + log, + }); - await seedCatalog({ - api: fakeAPI(handler), - cookies: [], - tenantId: TENANT_ID, - apiKey: "sk-test", - log, - }); + const priorities = offeringPosts.map((offering) => offering.priority); + const sonnet = offeringPosts.find( + (offering) => modelNamesById.get(offering.modelId) === "claude-sonnet-5", + ); + expect(new Set(priorities).size).toBe(offeringPosts.length); + expect(sonnet?.priority).toBe(Math.min(...priorities)); + }); - expect(patchedOfferings).toEqual([{ id: "off_legacy", priority: 1 }]); - }, - ); + test("re-run updates a legacy offering to its computed priority", async () => { + const { log } = collector(); + const patchedOfferings: { id: string; priority: number }[] = []; + const handler: FakeHandler = (method, path, body) => { + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) + return { status: 201, data: providerRow("prv_1", "anthropic") }; + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/credentials`) + return { + status: 201, + data: credentialRow("cre_1", "prv_1", "anthropic-default"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/models` + ) { + const canonicalName = (body as { canonicalName: string }).canonicalName; + const modelId = + canonicalName === "claude-opus-5" ? "mdl_legacy" : "mdl_new"; + return { status: 201, data: catalogModelRow(modelId, canonicalName) }; + } + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/providers` + ) + return { + status: 201, + data: catalogProviderRow("cpv_1", "anthropic", "cre_1"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + const offering = body as { modelId: string }; + if (offering.modelId === "mdl_legacy") + return { status: 409, data: { error: "already exists" } }; + return { + status: 201, + data: catalogOfferingRow( + `off_${offering.modelId}`, + offering.modelId, + "cpv_1", + ), + }; + } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) + return { + status: 200, + data: { + data: [ + catalogOfferingRow( + "off_other_provider", + "mdl_legacy", + "cpv_other", + ), + ], + nextCursor: "second-page", + }, + }; + if ( + method === "GET" && + path === + `/api/tenants/${TENANT_ID}/catalog/offerings?cursor=second-page` + ) + return { + status: 200, + data: { + data: [catalogOfferingRow("off_legacy", "mdl_legacy", "cpv_1")], + nextCursor: null, + }, + }; + if ( + method === "PATCH" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings/off_legacy` + ) { + const patch = body as { priority: number }; + patchedOfferings.push({ id: "off_legacy", priority: patch.priority }); + return { + status: 200, + data: { + ...catalogOfferingRow("off_legacy", "mdl_legacy", "cpv_1"), + priority: patch.priority, + }, + }; + } + return undefined; + }; + + await seedCatalog({ + api: fakeAPI(handler), + cookies: [], + tenantId: TENANT_ID, + apiKey: "sk-test", + log, + }); + + expect(patchedOfferings).toEqual([{ id: "off_legacy", priority: 1 }]); + }); test("an Ollama offering's quirks carry that model's real context-window ceiling, not the built-in 4096 default", async () => { const { log } = collector(); @@ -1903,6 +1888,7 @@ describe("seedCatalog", () => { const { lines, log } = collector(); let patchCalls = 0; let postCredentialCalls = 0; + let offeringPosts = 0; let patchBody: unknown; const staleCredentialRow = () => ({ @@ -1975,11 +1961,17 @@ describe("seedCatalog", () => { method === "POST" && path === `/api/tenants/${TENANT_ID}/catalog/offerings` ) { - // Priority = the provider's CATALOG_SEEDS declaration index, so - // multi-provider fallback order is deterministic, never a tie. - expect((body as { priority: number }).priority).toBe( + const providersBeforeHuggingFace = Object.entries(CATALOG_SEEDS).slice( + 0, Object.keys(CATALOG_SEEDS).indexOf("huggingface"), ); + expect((body as { priority: number }).priority).toBe( + providersBeforeHuggingFace.reduce( + (offset, [, providerSeed]) => offset + providerSeed.models.length, + 0, + ) + offeringPosts, + ); + offeringPosts += 1; return { status: 201, data: catalogOfferingRow("off_1", "mdl_1", "cpv_1"), @@ -2380,6 +2372,24 @@ describe("seedCatalog", () => { offeringPosts += 1; return { status: 409, data: { error: "already exists" } }; } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) + return { + status: 200, + data: { + data: anthropicModels.map((_, index) => ({ + ...catalogOfferingRow( + `off_${index + 1}`, + `mdl_${index + 1}`, + "cpv_1", + ), + priority: index, + })), + nextCursor: null, + }, + }; return undefined; };