From 080248c04c76d3fcf376241f5315bcc1162e101b Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:41:17 +0000 Subject: [PATCH] feat: Move `projectGroupName` to `projects.create()` --- .changeset/project-group-name.md | 2 +- js/src/framework.ts | 12 +- js/src/framework2.test.ts | 47 +++++- js/src/framework2.ts | 20 ++- js/src/logger.ts | 39 +---- js/src/project-group.test.ts | 255 ------------------------------- 6 files changed, 64 insertions(+), 311 deletions(-) delete mode 100644 js/src/project-group.test.ts diff --git a/.changeset/project-group-name.md b/.changeset/project-group-name.md index eeae4b701..5b2425b5e 100644 --- a/.changeset/project-group-name.md +++ b/.changeset/project-group-name.md @@ -4,4 +4,4 @@ feat: Add `projectGroupName` to create projects inside a project group -`initLogger`, `init`, `initDataset`, and `Eval` now accept `projectGroupName`. When the named project does not exist yet, it is created inside that project group, which lets callers who only hold project-creation permission on a group (rather than on the whole organization) register projects. The option is ignored when a `projectId` is supplied. +`projects.create({ name, projectGroupName })` now accepts a project group name. When `braintrust push` or `project.publish()` registers a project that does not exist yet, it is created inside that project group, which lets callers who only hold project-creation permission on a group (rather than on the whole organization) register projects. diff --git a/js/src/framework.ts b/js/src/framework.ts index e60f3d2dc..fa18da27c 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -327,13 +327,6 @@ export interface Evaluator< */ projectId?: string; - /** - * If specified, creates the project inside the project group with this name when the project does - * not already exist. Requires permission to create projects in that group. Ignored if `projectId` - * is specified. - */ - projectGroupName?: string; - /** * If specified, uses the logger state to initialize Braintrust objects. If unspecified, falls back * to the global state (initialized using your API key). @@ -522,10 +515,7 @@ export async function _internalInitEvaluatorExperiment( return initExperiment(evaluator.state, { ...(evaluator.projectId ? { projectId: evaluator.projectId } - : { - project: projectName, - projectGroupName: evaluator.projectGroupName, - }), + : { project: projectName }), experiment: options.experimentName ?? evaluator.experimentName, description: evaluator.description, metadata: evaluator.metadata, diff --git a/js/src/framework2.test.ts b/js/src/framework2.test.ts index 2c8acfe65..35297ab36 100644 --- a/js/src/framework2.test.ts +++ b/js/src/framework2.test.ts @@ -1,11 +1,13 @@ -import { beforeAll, expect, expectTypeOf, test } from "vitest"; +import { afterEach, beforeAll, expect, expectTypeOf, test, vi } from "vitest"; import * as z3 from "zod/v3"; import * as z4 from "zod/v4"; import { CodeFunction, + ProjectNameIdMap, projects, serializeEvalParametersToStaticParametersSchema, } from "./framework2"; +import { _internalGetGlobalState } from "./logger"; import { configureNode } from "./node/config"; import { zodToJsonSchema } from "./zod/utils"; @@ -13,6 +15,49 @@ beforeAll(() => { configureNode(); }); +afterEach(() => { + vi.restoreAllMocks(); +}); + +function mockProjectRegister() { + const postJson = vi.fn(async () => ({ + project: { + id: "00000000-0000-0000-0000-000000000001", + org_id: "00000000-0000-0000-0000-000000000002", + name: "project", + }, + })); + vi.spyOn(_internalGetGlobalState(), "appConn").mockReturnValue({ + post_json: postJson, + } as never); + return postJson; +} + +test("project resolution creates the project in its project group", async () => { + const postJson = mockProjectRegister(); + const project = projects.create({ + name: "project", + projectGroupName: "my-group", + }); + + expect(await new ProjectNameIdMap().resolve(project)).toBe( + "00000000-0000-0000-0000-000000000001", + ); + expect(postJson).toHaveBeenCalledWith("api/project/register", { + project_name: "project", + project_group_name: "my-group", + }); +}); + +test("project resolution omits project_group_name when unspecified", async () => { + const postJson = mockProjectRegister(); + + await new ProjectNameIdMap().resolve(projects.create({ name: "project" })); + expect(postJson).toHaveBeenCalledWith("api/project/register", { + project_name: "project", + }); +}); + test.each([ { version: "v3", diff --git a/js/src/framework2.ts b/js/src/framework2.ts index b65d96bdd..22fff00ac 100644 --- a/js/src/framework2.ts +++ b/js/src/framework2.ts @@ -43,9 +43,16 @@ export { toolFunctionDefinitionSchema }; const currentFilename = typeof __filename !== "undefined" ? __filename : "unknown"; -type NameOrId = { name: string } | { id: string }; - -export type CreateProjectOpts = NameOrId; +export type CreateProjectOpts = + | { + name: string; + /** + * If specified, creates the project inside the project group with this name when the project + * does not already exist. Requires permission to create projects in that group. + */ + projectGroupName?: string; + } + | { id: string }; class ProjectBuilder { create(opts: CreateProjectOpts) { return new Project(opts); @@ -56,6 +63,7 @@ export const projects = new ProjectBuilder(); export class Project { public readonly name?: string; public readonly id?: string; + public readonly projectGroupName?: string; public tools: ToolBuilder; public prompts: PromptBuilder; public parameters: ParametersBuilder; @@ -77,6 +85,7 @@ export class Project { _initializeSpanContext(); this.name = "name" in args ? args.name : undefined; this.id = "id" in args ? args.id : undefined; + this.projectGroupName = "name" in args ? args.projectGroupName : undefined; this.tools = new ToolBuilder(this); this.prompts = new PromptBuilder(this); this.parameters = new ParametersBuilder(this); @@ -857,12 +866,13 @@ export class ProjectNameIdMap { private nameToId: Record = {}; private idToName: Record = {}; - async getId(projectName: string): Promise { + async getId(projectName: string, projectGroupName?: string): Promise { if (!(projectName in this.nameToId)) { const response = await _internalGetGlobalState() .appConn() .post_json("api/project/register", { project_name: projectName, + ...(projectGroupName ? { project_group_name: projectGroupName } : {}), }); const result = z @@ -898,6 +908,6 @@ export class ProjectNameIdMap { if (project.id) { return project.id; } - return this.getId(project.name!); + return this.getId(project.name!, project.projectGroupName); } } diff --git a/js/src/logger.ts b/js/src/logger.ts index a6fb792c6..ce98c1be8 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -4159,7 +4159,6 @@ export type InitOptions = FullLoginOptions & { tags?: string[]; gitMetadataSettings?: GitMetadataSettings; projectId?: string; - projectGroupName?: string; baseExperimentId?: string; repoInfo?: RepoInfo; setCurrent?: boolean; @@ -4218,7 +4217,6 @@ type InitializedExperiment = * @param setCurrent If true (the default), set the global current-experiment to the newly-created one. * @param options.open If the experiment already exists, open it in read-only mode. Throws an error if the experiment does not already exist. * @param options.projectId The id of the project to create the experiment in. This takes precedence over `project` if specified. - * @param options.projectGroupName (Optional) Create the project inside the project group with this name, if the project does not already exist. Requires permission to create projects in that group. Ignored if `projectId` is specified. * @param options.baseExperimentId An optional experiment id to use as a base. If specified, the new experiment will be summarized and compared to this. This takes precedence over `baseExperiment` if specified. * @param options.repoInfo (Optional) Explicitly specify the git metadata for this experiment. This takes precedence over `gitMetadataSettings` if specified. * @param options.tags (Optional) A list of tags to attach to the experiment. @@ -4278,7 +4276,6 @@ export function init( tags, gitMetadataSettings, projectId, - projectGroupName, baseExperimentId, repoInfo, state: stateArg, @@ -4358,18 +4355,6 @@ export function init( update, }; - if (!isEmpty(projectGroupName) && isEmpty(projectId)) { - // `api/experiment/register` cannot create a project inside a project - // group, so resolve the project (creating it in the group if needed) - // up front and register the experiment against its id. - const { project: groupProject } = await computeLoggerMetadata(state, { - project_name: project, - project_group_name: projectGroupName, - }); - delete args["project_name"]; - args["project_id"] = groupProject.id; - } - if (experiment) { args["experiment_name"] = experiment; } @@ -4591,7 +4576,6 @@ export type InitDatasetOptions = environment?: string; snapshotName?: string; projectId?: string; - projectGroupName?: string; metadata?: Record; state?: BraintrustState; _internal_btql?: Record; @@ -4883,7 +4867,6 @@ export function initDataset< fetch, forceLogin, projectId, - projectGroupName, metadata, useOutput: legacy, state: stateArg, @@ -4968,19 +4951,6 @@ export function initDataset< description, metadata, }; - - if (!isEmpty(projectGroupName) && isEmpty(projectId)) { - // `api/dataset/register` cannot create a project inside a project - // group, so resolve the project (creating it in the group if needed) - // up front and register the dataset against its id. - const { project: groupProject } = await computeLoggerMetadata(state, { - project_name: project, - project_group_name: projectGroupName, - }); - delete args["project_name"]; - args["project_id"] = groupProject.id; - } - const response = await state .appConn() .post_json("api/dataset/register", args); @@ -5081,11 +5051,9 @@ async function computeLoggerMetadata( { project_name, project_id, - project_group_name, }: { project_name?: string; project_id?: string; - project_group_name?: string; }, ) { await state.login({}); @@ -5103,7 +5071,7 @@ async function computeLoggerMetadata( org_id, state.loginToken, isEmpty(project_id) - ? ["name", project_name || GLOBAL_PROJECT, project_group_name ?? null] + ? ["name", project_name || GLOBAL_PROJECT] : ["id", project_id], ]); const cached = cache.get(key); @@ -5125,7 +5093,6 @@ async function computeLoggerMetadata( { project_name: project_name || GLOBAL_PROJECT, org_id, - ...(isEmpty(project_group_name) ? {} : { project_group_name }), }, controller.signal, ); @@ -5186,7 +5153,6 @@ type AsyncFlushArg = { export type InitLoggerOptions = FullLoginOptions & { projectName?: string; projectId?: string; - projectGroupName?: string; environment?: SpanOriginEnvironment; setCurrent?: boolean; state?: BraintrustState; @@ -5199,7 +5165,6 @@ export type InitLoggerOptions = FullLoginOptions & { * @param options Additional options for configuring init(). * @param options.projectName The name of the project to log into. If unspecified, will default to the Global project. * @param options.projectId The id of the project to log into. This takes precedence over projectName if specified. - * @param options.projectGroupName (Optional) Create the project inside the project group with this name, if the project does not already exist. Requires permission to create projects in that group. Ignored if `projectId` is specified. * @param options.asyncFlush If true, will log asynchronously in the background. Otherwise, will log synchronously. (true by default) * @param options.appUrl The URL of the Braintrust App. Defaults to https://www.braintrust.dev. * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. In Node.js, @@ -5216,7 +5181,6 @@ export function initLogger( const { projectName, projectId, - projectGroupName, asyncFlush: asyncFlushArg, appUrl, apiKey, @@ -5234,7 +5198,6 @@ export function initLogger( const computeMetadataArgs = { project_name: projectName, project_id: projectId, - project_group_name: projectGroupName, }; const linkArgs = { diff --git a/js/src/project-group.test.ts b/js/src/project-group.test.ts deleted file mode 100644 index f8efb646d..000000000 --- a/js/src/project-group.test.ts +++ /dev/null @@ -1,255 +0,0 @@ -import { afterEach, describe, expect, test, vi } from "vitest"; -import { - BraintrustState, - init, - initDataset, - initLogger, - spanComponentsToObjectId, -} from "./logger"; -import { SpanComponentsV4 } from "../util/span_identifier_v4"; -import { _internalInitEvaluatorExperiment } from "./framework"; -import { configureNode } from "./node/config"; - -configureNode(); - -const projectId = "00000000-0000-0000-0000-000000000001"; - -function createState() { - const fetch = vi.fn(async (input) => { - const url = new URL(String(input)); - switch (url.pathname) { - case "/api/apikey/login": - return Response.json({ - org_info: [ - { id: "org-id", name: "org-id", api_url: "https://api.test" }, - ], - }); - case "/api/project/register": - return Response.json({ project: { id: projectId, name: "project" } }); - case "/api/experiment/register": - return Response.json({ - project: { id: projectId, name: "project" }, - experiment: { id: "experiment-id", name: "experiment" }, - }); - case "/api/dataset/register": - return Response.json({ - project: { id: projectId, name: "project" }, - dataset: { id: "dataset-id", name: "dataset" }, - }); - case "/version": - case "/logs3": - return Response.json({}); - default: - throw new Error(`Unexpected test request: ${url.pathname}`); - } - }); - const state = new BraintrustState({ - apiKey: "test-credential", - appUrl: "https://app.test", - fetch, - noExitFlush: true, - }); - return { state, fetch }; -} - -function bodiesFor( - fetch: ReturnType["fetch"], - pathname: string, -): Record[] { - return fetch.mock.calls - .filter(([url]) => new URL(String(url)).pathname === pathname) - .map(([, init]) => JSON.parse(String(init?.body))); -} - -afterEach(() => { - vi.restoreAllMocks(); -}); - -describe("projectGroupName", () => { - test("initLogger forwards project_group_name to project registration", async () => { - const { state, fetch } = createState(); - expect( - await initLogger({ - state, - projectName: "project", - projectGroupName: "my-group", - setCurrent: false, - }).id, - ).toBe(projectId); - - expect(bodiesFor(fetch, "/api/project/register")).toEqual([ - { - project_name: "project", - org_id: "org-id", - project_group_name: "my-group", - }, - ]); - }); - - test("initLogger omits project_group_name when unspecified", async () => { - const { state, fetch } = createState(); - await initLogger({ - state, - projectName: "project", - setCurrent: false, - }).id; - - const [body] = bodiesFor(fetch, "/api/project/register"); - expect(body).not.toHaveProperty("project_group_name"); - }); - - test("project metadata cache keys on the project group name", async () => { - const { state, fetch } = createState(); - const base = { state, projectName: "project", setCurrent: false }; - await initLogger({ ...base, projectGroupName: "group-a" }).id; - await initLogger({ ...base, projectGroupName: "group-a" }).id; - await initLogger({ ...base, projectGroupName: "group-b" }).id; - await initLogger(base).id; - - expect( - bodiesFor(fetch, "/api/project/register").map( - (body) => body.project_group_name, - ), - ).toEqual(["group-a", "group-b", undefined]); - }); - - test("init creates the project in the group, then registers by id", async () => { - const { state, fetch } = createState(); - const experiment = init({ - state, - project: "project", - projectGroupName: "my-group", - experiment: "experiment", - setCurrent: false, - baseExperimentId: "base-experiment-id", - repoInfo: {}, - }); - expect(await experiment.id).toBe("experiment-id"); - - expect(bodiesFor(fetch, "/api/project/register")).toEqual([ - { - project_name: "project", - org_id: "org-id", - project_group_name: "my-group", - }, - ]); - const [body] = bodiesFor(fetch, "/api/experiment/register"); - expect(body).toMatchObject({ project_id: projectId }); - expect(body).not.toHaveProperty("project_name"); - }); - - test("init registers directly when no project group is given", async () => { - const { state, fetch } = createState(); - await init({ - state, - project: "project", - experiment: "experiment", - setCurrent: false, - baseExperimentId: "base-experiment-id", - repoInfo: {}, - }).id; - - expect(bodiesFor(fetch, "/api/project/register")).toEqual([]); - expect(bodiesFor(fetch, "/api/experiment/register")[0]).toMatchObject({ - project_name: "project", - }); - }); - - test("initDataset creates the project in the group, then registers by id", async () => { - const { state, fetch } = createState(); - const dataset = initDataset({ - state, - project: "project", - projectGroupName: "my-group", - dataset: "dataset", - }); - expect(await dataset.id).toBe("dataset-id"); - - expect(bodiesFor(fetch, "/api/project/register")).toEqual([ - { - project_name: "project", - org_id: "org-id", - project_group_name: "my-group", - }, - ]); - const [body] = bodiesFor(fetch, "/api/dataset/register"); - expect(body).toMatchObject({ project_id: projectId }); - expect(body).not.toHaveProperty("project_name"); - }); - - test("projectGroupName is ignored when projectId is specified", async () => { - const { state, fetch } = createState(); - await initDataset({ - state, - projectId, - projectGroupName: "my-group", - dataset: "dataset", - }).id; - - expect(bodiesFor(fetch, "/api/project/register")).toEqual([]); - expect(bodiesFor(fetch, "/api/dataset/register")[0]).toMatchObject({ - project_id: projectId, - }); - }); - - test("Eval creates the project in the group before the experiment", async () => { - const { state, fetch } = createState(); - const experiment = await _internalInitEvaluatorExperiment( - "project", - { - projectName: "project", - projectGroupName: "my-group", - evalName: "eval", - data: [], - task: (input: unknown) => input, - scores: [], - state, - baseExperimentId: "base-experiment-id", - repoInfo: {}, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any, - [], - ); - expect(await experiment?.id).toBe("experiment-id"); - - expect(bodiesFor(fetch, "/api/project/register")).toEqual([ - { - project_name: "project", - org_id: "org-id", - project_group_name: "my-group", - }, - ]); - expect(bodiesFor(fetch, "/api/experiment/register")[0]).toMatchObject({ - project_id: projectId, - }); - }); - - test("exported span components carry the project group for lazy resolution", async () => { - const { state, fetch } = createState(); - const logger = initLogger({ - state, - projectName: "project", - projectGroupName: "my-group", - setCurrent: false, - }); - - // The logger has not resolved its id yet, so `export()` defers project - // resolution (and creation) to whoever consumes the exported components. - const components = SpanComponentsV4.fromStr(await logger.export()); - expect(components.data.compute_object_metadata_args).toMatchObject({ - project_name: "project", - project_group_name: "my-group", - }); - - expect(await spanComponentsToObjectId({ components, state })).toBe( - projectId, - ); - expect(bodiesFor(fetch, "/api/project/register")).toEqual([ - { - project_name: "project", - org_id: "org-id", - project_group_name: "my-group", - }, - ]); - }); -});