From 4cb34df177914588a9272d9970a646cc0b12f643 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 8 Sep 2026 14:31:10 -0400 Subject: [PATCH 1/3] feat(api): generate and migrate Prompt REST resources ### AI Summary Implements the Prompt slice of #683 by publishing all six operations tagged Prompts and moving `load_prompt()` retrieval onto the public REST resources. Prompt request and response models are also exported through the public API. ### Migration flow ```text Before load_prompt(id=...) ------> raw GET /v1/prompt/{id} load_prompt(slug=...) ----> raw GET /v1/prompt After load_prompt(id=...) ------> generated GET /v1/prompt/{id} load_prompt(slug=...) ----> generated GET /v1/prompt ``` | SDK workflow | Previous wire call | Public REST call | | --- | --- | --- | | Load by ID | Raw `GET /v1/prompt/{id}` | Generated `GET /v1/prompt/{id}` | | Load by project and slug | Raw `GET /v1/prompt` | Generated `GET /v1/prompt` | | Prompt CRUD | No generated resource | `client.openapi.prompts` | | Behavior | Result | | --- | --- | | Explicit credentials | Use isolated, cached generated clients without mutating global login | | Active global login | Reuse the existing generated API client | | Version and environment | Explicit version continues to take precedence over environment | | Cache fallback | Preserve local prompt cache fallback for retryable server failures | | Lazy and async loading | Continue deferring retrieval until the prompt is used | | Prompt lookup by ID | Consume the direct response without a synthetic `objects` envelope | | Retry behavior | GET operations are safe reads; version-producing writes are not retried | VCR coverage exercises generated CRUD against the real backend and verifies that high-level prompt loading uses the generated resources. Wire, model, packaging, and type tests cover the public bindings. Refs #683 --- openapi/README.md | 2 +- openapi/config.json | 3 +- .../api/_generated/models/__init__.py | 134 ++++- .../api/_generated/models/prompts.py | 534 +++++++++++++++++ py/src/braintrust/api/_generated/prompts.py | 316 ++++++++++ ..._load_prompt_uses_generated_resources.yaml | 510 ++++++++++++++++ ..._prompts_end_to_end_with_real_backend.yaml | 552 ++++++++++++++++++ py/src/braintrust/api/client.py | 2 + .../braintrust/api/test_generated_models.py | 8 +- py/src/braintrust/api/test_prompts.py | 152 +++++ py/src/braintrust/api/types/__init__.py | 8 + py/src/braintrust/logger.py | 123 ++-- py/src/braintrust/test_logger.py | 134 +++-- .../braintrust/type_tests/test_api_client.py | 26 + 14 files changed, 2409 insertions(+), 95 deletions(-) create mode 100644 py/src/braintrust/api/_generated/models/prompts.py create mode 100644 py/src/braintrust/api/_generated/prompts.py create mode 100644 py/src/braintrust/api/cassettes/test_high_level_load_prompt_uses_generated_resources.yaml create mode 100644 py/src/braintrust/api/cassettes/test_prompts_end_to_end_with_real_backend.yaml create mode 100644 py/src/braintrust/api/test_prompts.py diff --git a/openapi/README.md b/openapi/README.md index bafcaac4..73ba5446 100644 --- a/openapi/README.md +++ b/openapi/README.md @@ -16,7 +16,7 @@ make check-api-client-codegen ``` The check regenerates in a temporary directory and reports drift without changing the worktree. -Currently selected tags are Projects, Experiments, and Datasets. Each tag produces one resource and +Currently selected tags are Projects, Experiments, Datasets, and Prompts. Each tag produces one resource and operation registry. Models used by one resource stay in that resource's model module; shared models live in `models/common.py`; unreachable models are omitted. diff --git a/openapi/config.json b/openapi/config.json index 8ce3da29..7c11e179 100644 --- a/openapi/config.json +++ b/openapi/config.json @@ -32,7 +32,8 @@ "generated_tags": [ "Projects", "Experiments", - "Datasets" + "Datasets", + "Prompts" ], "safe_reads": [ "postExperimentIdFetch", diff --git a/py/src/braintrust/api/_generated/models/__init__.py b/py/src/braintrust/api/_generated/models/__init__.py index 5006f61a..4978c916 100644 --- a/py/src/braintrust/api/_generated/models/__init__.py +++ b/py/src/braintrust/api/_generated/models/__init__.py @@ -4,7 +4,7 @@ # datamodel-code-generator: 0.72.4 # ruff: 0.15.21 # Generator Python: 3.14 -# Content SHA-256: 1ac0980e6a11327b27716b8975b8953969d1c95dd4ed5ac4b9d8118258a9f4bc +# Content SHA-256: 494c7e1ba491afef034f77e2f459fc17c49217b4057b10ab3abd2b5982ff48be """Generated private model types with stable package-level imports.""" @@ -90,17 +90,102 @@ RemoteEvalSource, SpanFieldOrderItem, ) +from .prompts import ( + CacheControl, + ChatCompletionContentPart, + ChatCompletionContentPartFileFile, + ChatCompletionContentPartFileWithTitle, + ChatCompletionContentPartImageWithTitle, + ChatCompletionContentPartText, + ChatCompletionContentPartTextWithTitle, + ChatCompletionMessageParam, + ChatCompletionMessageParam1, + ChatCompletionMessageParam2, + ChatCompletionMessageParam3, + ChatCompletionMessageParam4, + ChatCompletionMessageParam5, + ChatCompletionMessageParam6, + ChatCompletionMessageParam7, + ChatCompletionMessageReasoning, + ChatCompletionMessageToolCall, + CreatePrompt, + Function, + Function1, + FunctionCall, + FunctionCall1, + FunctionTypeEnumNullish, + GetPromptResponse, + ImageUrl, + Mcp, + Mcp1, + ModelParams, + ModelParams1, + ModelParams2, + ModelParams3, + ModelParams4, + ModelParams5, + Origin, + PatchPrompt, + PreprocessorId, + PreprocessorId1, + PreprocessorId2, + PreprocessorId3, + Prompt, + PromptBlockDataNullish, + PromptBlockDataNullish1, + PromptBlockDataNullish2, + PromptDataNullish, + PromptEnvironment, + PromptIdParam, + PromptName, + PromptOptionsNullish, + PromptParserNullish, + PromptVersion, + ResponseFormatJsonSchema, + ResponseFormatNullish, + ResponseFormatNullish1, + ResponseFormatNullish2, + ResponseFormatNullish3, + Slug, + ToolChoice, + ToolFunction, + ToolFunction1, + ToolFunction2, + ToolFunction3, + ToolFunction4, + ToolFunction5, + ToolFunction6, + ToolFunction7, +) __all__ = [ "AppLimitParam", "AppLimitWithDefaultParam", + "CacheControl", + "ChatCompletionContentPart", + "ChatCompletionContentPartFileFile", + "ChatCompletionContentPartFileWithTitle", + "ChatCompletionContentPartImageWithTitle", + "ChatCompletionContentPartText", + "ChatCompletionContentPartTextWithTitle", + "ChatCompletionMessageParam", + "ChatCompletionMessageParam1", + "ChatCompletionMessageParam2", + "ChatCompletionMessageParam3", + "ChatCompletionMessageParam4", + "ChatCompletionMessageParam5", + "ChatCompletionMessageParam6", + "ChatCompletionMessageParam7", + "ChatCompletionMessageReasoning", + "ChatCompletionMessageToolCall", "Classification", "ComparisonExperimentId", "Context", "CreateDataset", "CreateExperiment", "CreateProject", + "CreatePrompt", "DataSummary", "Dataset", "DatasetEvent", @@ -123,11 +208,18 @@ "FetchLimitParam", "FetchPaginationCursor", "FieldArrayDeleteItem", + "Function", + "Function1", + "FunctionCall", + "FunctionCall1", "FunctionTypeEnum", + "FunctionTypeEnumNullish", "GetDatasetResponse", "GetExperimentResponse", "GetProjectResponse", + "GetPromptResponse", "Ids", + "ImageUrl", "InsertDatasetEvent", "InsertDatasetEventRequest", "InsertEventsResponse", @@ -136,28 +228,59 @@ "InternalMetadata", "MaxRootSpanId", "MaxXactId", + "Mcp", + "Mcp1", "Metadata", "MetricSummary", "Metrics", + "ModelParams", + "ModelParams1", + "ModelParams2", + "ModelParams3", + "ModelParams4", + "ModelParams5", "NullableSavedFunctionId", "NullableSavedFunctionId1", "NullableSavedFunctionId2", "ObjectReferenceNullish", "OrgName", + "Origin", "PatchDataset", "PatchExperiment", "PatchProject", + "PatchPrompt", + "PreprocessorId", + "PreprocessorId1", + "PreprocessorId2", + "PreprocessorId3", "Project", "ProjectIdParam", "ProjectIdQuery", "ProjectName", "ProjectSettings", + "Prompt", + "PromptBlockDataNullish", + "PromptBlockDataNullish1", + "PromptBlockDataNullish2", + "PromptDataNullish", + "PromptEnvironment", + "PromptIdParam", + "PromptName", + "PromptOptionsNullish", + "PromptParserNullish", + "PromptVersion", "RemoteEvalSource", "RepoInfo", + "ResponseFormatJsonSchema", + "ResponseFormatNullish", + "ResponseFormatNullish1", + "ResponseFormatNullish2", + "ResponseFormatNullish3", "SavedFunctionId", "SavedFunctionId1", "SavedFunctionId2", "ScoreSummary", + "Slug", "SpanAttributes", "SpanFieldOrderItem", "SpanType", @@ -166,5 +289,14 @@ "SummarizeDatasetResponse", "SummarizeExperimentResponse", "SummarizeScores", + "ToolChoice", + "ToolFunction", + "ToolFunction1", + "ToolFunction2", + "ToolFunction3", + "ToolFunction4", + "ToolFunction5", + "ToolFunction6", + "ToolFunction7", "Version", ] diff --git a/py/src/braintrust/api/_generated/models/prompts.py b/py/src/braintrust/api/_generated/models/prompts.py new file mode 100644 index 00000000..850dc4f0 --- /dev/null +++ b/py/src/braintrust/api/_generated/models/prompts.py @@ -0,0 +1,534 @@ +# Generated by scripts/generate-api-client.py. DO NOT EDIT. +# OpenAPI commit: 8d6e24ce78d8e33cbf9ba5f35d2fbc71749306d7 +# OpenAPI spec SHA-256: ebac33c1422e5673f875a254b3f7a13e4d8b5f5ab9b5095fd6b48e836613d507 +# datamodel-code-generator: 0.72.4 +# ruff: 0.15.21 +# Generator Python: 3.14 +# Content SHA-256: bacb6ce9b0a44df81cd41c75e92a79000c50c826bee226218b541ea94dd73636 + +from typing import Any, Literal, TypeAlias, TypedDict +from typing_extensions import NotRequired +from collections.abc import Mapping, Sequence + +from .common import FunctionTypeEnum + + +class ChatCompletionContentPartFileFile(TypedDict): + file_data: NotRequired[str] + file_id: NotRequired[str] + filename: NotRequired[str] + + +class CacheControl(TypedDict): + ttl: NotRequired[Literal["5m", "1h"]] + type: Literal["ephemeral"] + + +class ChatCompletionContentPartFileWithTitle(TypedDict): + cache_control: NotRequired[CacheControl] + file: ChatCompletionContentPartFileFile + type: Literal["file"] + + +class ImageUrl(TypedDict): + detail: NotRequired[Literal["auto"] | Literal["low"] | Literal["high"]] + url: str + + +class ChatCompletionContentPartImageWithTitle(TypedDict): + cache_control: NotRequired[CacheControl] + image_url: ImageUrl + type: Literal["image_url"] + + +class ChatCompletionContentPartText(TypedDict): + cache_control: NotRequired[CacheControl] + text: NotRequired[str] + type: Literal["text"] + + +class ChatCompletionContentPartTextWithTitle(TypedDict): + cache_control: NotRequired[CacheControl] + text: NotRequired[str] + type: Literal["text"] + + +class ChatCompletionMessageParam1(TypedDict): + content: NotRequired[str | Sequence[ChatCompletionContentPartText]] + name: NotRequired[str] + role: Literal["system"] + + +class FunctionCall(TypedDict): + arguments: str + name: str + + +class ChatCompletionMessageParam4(TypedDict): + content: NotRequired[str | Sequence[ChatCompletionContentPartText]] + role: Literal["tool"] + tool_call_id: NotRequired[str] + + +class ChatCompletionMessageParam5(TypedDict): + content: str | None + name: str + role: Literal["function"] + + +class ChatCompletionMessageParam6(TypedDict): + content: NotRequired[str | Sequence[ChatCompletionContentPartText]] + name: NotRequired[str] + role: Literal["developer"] + + +class ChatCompletionMessageParam7(TypedDict): + content: NotRequired[str | None] + role: Literal["model"] + + +class ChatCompletionMessageReasoning(TypedDict): + content: NotRequired[str | None] + id: NotRequired[str | None] + + +class Function(TypedDict): + arguments: str + name: str + + +class ChatCompletionMessageToolCall(TypedDict): + function: Function + id: str + type: Literal["function"] + + +FunctionTypeEnumNullish: TypeAlias = ( + Literal[ + "llm", + "scorer", + "task", + "tool", + "custom_view", + "preprocessor", + "facet", + "classifier", + "tag", + "parameters", + "sandbox", + ] + | None +) + + +class FunctionCall1(TypedDict): + name: str + + +class Function1(TypedDict): + name: str + + +class ToolChoice(TypedDict): + function: Function1 + type: Literal["function"] + + +class ModelParams2(TypedDict): + max_tokens: float + max_tokens_to_sample: NotRequired[float] + """ + This is a legacy parameter that should not be used. + """ + reasoning_budget: NotRequired[float] + reasoning_enabled: NotRequired[bool] + stop_sequences: NotRequired[Sequence[str]] + temperature: float + top_k: NotRequired[float] + top_p: NotRequired[float] + use_cache: NotRequired[bool] + + +class ModelParams3(TypedDict): + maxOutputTokens: NotRequired[float] + reasoning_budget: NotRequired[float] + reasoning_enabled: NotRequired[bool] + temperature: NotRequired[float] + topK: NotRequired[float] + topP: NotRequired[float] + use_cache: NotRequired[bool] + + +class ModelParams4(TypedDict): + reasoning_budget: NotRequired[float] + reasoning_enabled: NotRequired[bool] + temperature: NotRequired[float] + topK: NotRequired[float] + use_cache: NotRequired[bool] + + +class ModelParams5(TypedDict): + reasoning_budget: NotRequired[float] + reasoning_enabled: NotRequired[bool] + use_cache: NotRequired[bool] + + +class PreprocessorId1(TypedDict): + id: str + type: Literal["function"] + version: NotRequired[str] + """ + The version of the function + """ + + +class PreprocessorId2(TypedDict): + function_type: NotRequired[Literal["preprocessor"]] + """ + The type of global function. Defaults to 'preprocessor'. + """ + name: str + type: Literal["global"] + + +class PreprocessorId3(TypedDict): + code: str + """ + The complete JavaScript preprocessor implementation, including its handler. + """ + type: Literal["inline"] + + +PreprocessorId: TypeAlias = PreprocessorId1 | PreprocessorId2 | PreprocessorId3 | None +""" +For prompt-backed functions: the saved, global, or inline preprocessor to use for trace template variables. Set to null to disable preprocessing. If omitted, the traced project's default preprocessor will be used, falling back to the global 'thread' preprocessor. +""" + + +class PromptBlockDataNullish2(TypedDict): + content: str + type: Literal["completion"] + + +class Mcp(TypedDict): + enabled_tools: NotRequired[Sequence[str] | None] + """ + If omitted, all tools are enabled + """ + id: str + is_disabled: NotRequired[bool] + type: Literal["id"] + + +class Mcp1(TypedDict): + enabled_tools: NotRequired[Sequence[str] | None] + """ + If omitted, all tools are enabled + """ + is_disabled: NotRequired[bool] + type: Literal["url"] + url: str + + +class Origin(TypedDict): + project_id: NotRequired[str] + prompt_id: NotRequired[str] + prompt_version: NotRequired[str] + + +class ToolFunction1(TypedDict): + id: str + type: Literal["function"] + version: NotRequired[str] + """ + The version of the function + """ + + +class ToolFunction2(TypedDict): + function_type: NotRequired[FunctionTypeEnum] + name: str + type: Literal["global"] + + +class ToolFunction3(TypedDict): + pass + + +class ToolFunction4(ToolFunction1, ToolFunction3): + pass + + +class ToolFunction5(ToolFunction2, ToolFunction3): + pass + + +class ToolFunction6(ToolFunction1, ToolFunction3): + pass + + +class ToolFunction7(ToolFunction2, ToolFunction3): + pass + + +ToolFunction: TypeAlias = ToolFunction4 | ToolFunction5 | ToolFunction6 | ToolFunction7 + +PromptEnvironment: TypeAlias = str +""" +Filter by environment slug. Cannot be used together with `version`. + +For `GET /v1/prompt`, environment resolution currently requires the request to match a single prompt. If multiple prompts match, the endpoint returns `400` (for example when `limit=1` is not set). Use `limit=1` or other filters (for example `slug`, `project_id`) to narrow results. +""" + +PromptIdParam: TypeAlias = str +""" +Prompt id +""" + +PromptName: TypeAlias = str +""" +Name of the prompt to search for +""" + + +class PromptParserNullish(TypedDict): + allow_no_match: NotRequired[bool] + """ + If true, adds a 'No match' option. When selected, no tag is deposited. + """ + allow_skip: NotRequired[bool] + """ + If true, adds a 'Skip' option. When selected, the scorer returns null. + """ + choice: NotRequired[Sequence[str]] + """ + List of valid choices without score mapping. Used by classifiers that deposit output to tags. + """ + choice_scores: NotRequired[Mapping[str, float]] + """ + Map of choices to scores (0-1). Used by scorers. + """ + type: Literal["llm_classifier"] + use_cot: bool + + +PromptVersion: TypeAlias = str +""" +Retrieve prompt at a specific version. + +The version id can either be a transaction id (e.g. '1000192656880881099') or a version identifier (e.g. '81cd05ee665fdfb3'). +""" + + +class ResponseFormatJsonSchema(TypedDict): + description: NotRequired[str] + name: str + schema: NotRequired[Mapping[str, Any] | str] + strict: NotRequired[bool | None] + + +class ResponseFormatNullish1(TypedDict): + type: Literal["json_object"] + + +class ResponseFormatNullish2(TypedDict): + json_schema: ResponseFormatJsonSchema + type: Literal["json_schema"] + + +class ResponseFormatNullish3(TypedDict): + type: Literal["text"] + + +ResponseFormatNullish: TypeAlias = ResponseFormatNullish1 | ResponseFormatNullish2 | ResponseFormatNullish3 | None + +Slug: TypeAlias = str +""" +Retrieve prompt with a specific slug +""" + +ChatCompletionContentPart: TypeAlias = ( + ChatCompletionContentPartTextWithTitle + | ChatCompletionContentPartImageWithTitle + | ChatCompletionContentPartFileWithTitle +) + + +class ChatCompletionMessageParam2(TypedDict): + content: NotRequired[str | Sequence[ChatCompletionContentPart]] + name: NotRequired[str] + role: Literal["user"] + + +class ChatCompletionMessageParam3(TypedDict): + content: NotRequired[str | Sequence[ChatCompletionContentPartText] | None] + function_call: NotRequired[FunctionCall | None] + name: NotRequired[str | None] + reasoning: NotRequired[Sequence[ChatCompletionMessageReasoning] | None] + reasoning_signature: NotRequired[str | None] + role: Literal["assistant"] + tool_calls: NotRequired[Sequence[ChatCompletionMessageToolCall] | None] + + +ChatCompletionMessageParam: TypeAlias = ( + ChatCompletionMessageParam1 + | ChatCompletionMessageParam2 + | ChatCompletionMessageParam3 + | ChatCompletionMessageParam4 + | ChatCompletionMessageParam5 + | ChatCompletionMessageParam6 + | ChatCompletionMessageParam7 +) + + +class ModelParams1(TypedDict): + frequency_penalty: NotRequired[float] + function_call: NotRequired[Literal["auto"] | Literal["none"] | FunctionCall1] + max_completion_tokens: NotRequired[float] + """ + The successor to max_tokens + """ + max_tokens: NotRequired[float] + n: NotRequired[float] + presence_penalty: NotRequired[float] + reasoning_budget: NotRequired[float] + reasoning_effort: NotRequired[Literal["none", "minimal", "low", "medium", "high"]] + reasoning_enabled: NotRequired[bool] + response_format: NotRequired[ResponseFormatNullish] + stop: NotRequired[Sequence[str]] + temperature: NotRequired[float] + tool_choice: NotRequired[Literal["auto"] | Literal["none"] | Literal["required"] | ToolChoice] + top_p: NotRequired[float] + use_cache: NotRequired[bool] + verbosity: NotRequired[Literal["low", "medium", "high"]] + + +ModelParams: TypeAlias = ModelParams1 | ModelParams2 | ModelParams3 | ModelParams4 | ModelParams5 + + +class PromptBlockDataNullish1(TypedDict): + messages: Sequence[ChatCompletionMessageParam] + tools: NotRequired[str] + type: Literal["chat"] + + +PromptBlockDataNullish: TypeAlias = PromptBlockDataNullish1 | PromptBlockDataNullish2 | None + + +class PromptOptionsNullish(TypedDict): + endpoint_name: NotRequired[str | None] + model: NotRequired[str] + params: NotRequired[ModelParams] + position: NotRequired[str] + + +class PromptDataNullish(TypedDict): + mcp: NotRequired[Mapping[str, Mcp | Mcp1] | None] + options: NotRequired[PromptOptionsNullish | None] + origin: NotRequired[Origin | None] + parser: NotRequired[PromptParserNullish | None] + preprocessor: NotRequired[PreprocessorId] + prompt: NotRequired[PromptBlockDataNullish] + template_format: NotRequired[Literal["mustache", "nunjucks", "none"] | None] + tool_functions: NotRequired[Sequence[ToolFunction] | None] + + +class CreatePrompt(TypedDict): + description: NotRequired[str | None] + """ + Textual description of the prompt + """ + function_type: NotRequired[FunctionTypeEnumNullish | None] + name: str + """ + Name of the prompt + """ + project_id: str + """ + Unique identifier for the project that the prompt belongs under + """ + prompt_data: NotRequired[PromptDataNullish | None] + slug: str + """ + Unique identifier for the prompt + """ + tags: NotRequired[Sequence[str] | None] + """ + A list of tags for the prompt + """ + + +class PatchPrompt(TypedDict): + description: NotRequired[str | None] + """ + Textual description of the prompt + """ + name: NotRequired[str | None] + """ + Name of the prompt + """ + prompt_data: NotRequired[PromptDataNullish | None] + slug: NotRequired[str | None] + """ + Unique identifier for the prompt + """ + tags: NotRequired[Sequence[str] | None] + """ + A list of tags for the prompt + """ + + +class Prompt(TypedDict): + _xact_id: str + """ + The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the prompt (see the `version` parameter) + """ + created: NotRequired[str | None] + """ + Date of prompt creation + """ + description: NotRequired[str | None] + """ + Textual description of the prompt + """ + function_type: NotRequired[FunctionTypeEnumNullish | None] + id: str + """ + Unique identifier for the prompt + """ + log_id: Literal["p"] + """ + A literal 'p' which identifies the object as a project prompt + """ + metadata: NotRequired[Mapping[str, Any] | None] + """ + User-controlled metadata about the prompt + """ + name: str + """ + Name of the prompt + """ + org_id: str + """ + Unique identifier for the organization + """ + project_id: str + """ + Unique identifier for the project that the prompt belongs under + """ + prompt_data: NotRequired[PromptDataNullish | None] + slug: str + """ + Unique identifier for the prompt + """ + tags: NotRequired[Sequence[str] | None] + """ + A list of tags for the prompt + """ + + +class GetPromptResponse(TypedDict): + objects: Sequence[Prompt] + """ + A list of prompt objects + """ diff --git a/py/src/braintrust/api/_generated/prompts.py b/py/src/braintrust/api/_generated/prompts.py new file mode 100644 index 00000000..816f240b --- /dev/null +++ b/py/src/braintrust/api/_generated/prompts.py @@ -0,0 +1,316 @@ +# Generated by scripts/generate-api-client.py. DO NOT EDIT. +# OpenAPI commit: 8d6e24ce78d8e33cbf9ba5f35d2fbc71749306d7 +# OpenAPI spec SHA-256: ebac33c1422e5673f875a254b3f7a13e4d8b5f5ab9b5095fd6b48e836613d507 +# datamodel-code-generator: 0.72.4 +# ruff: 0.15.21 +# Generator Python: 3.14 +# Content SHA-256: 2f8d43a3bacf5e304ec52fca1dd92b7c050bbdda4033df72c96c39d2a584380b + +"""Generated Prompts REST operations and resource.""" + +from typing import cast + +from .._service import Operation, Parameter, ResourceAPI +from ..policies import RetryMode +from .models.common import AppLimitParam, EndingBefore, Ids, OrgName, ProjectIdQuery, ProjectName, StartingAfter +from .models.prompts import ( + CreatePrompt, + GetPromptResponse, + PatchPrompt, + Prompt, + PromptEnvironment, + PromptIdParam, + PromptName, + PromptVersion, + Slug, +) + + +POST_PROMPT = Operation( + operation_id="postPrompt", + method="POST", + path="/v1/prompt", + parameters=(), + has_request_body=True, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.NONE, +) + + +PUT_PROMPT = Operation( + operation_id="putPrompt", + method="PUT", + path="/v1/prompt", + parameters=(), + has_request_body=True, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.NONE, +) + + +GET_PROMPT = Operation( + operation_id="getPrompt", + method="GET", + path="/v1/prompt", + parameters=( + Parameter( + argument_name="limit", + name="limit", + location="query", + required=False, + ), + Parameter( + argument_name="starting_after", + name="starting_after", + location="query", + required=False, + ), + Parameter( + argument_name="ending_before", + name="ending_before", + location="query", + required=False, + ), + Parameter( + argument_name="ids", + name="ids", + location="query", + required=False, + ), + Parameter( + argument_name="prompt_name", + name="prompt_name", + location="query", + required=False, + ), + Parameter( + argument_name="project_name", + name="project_name", + location="query", + required=False, + ), + Parameter( + argument_name="project_id", + name="project_id", + location="query", + required=False, + ), + Parameter( + argument_name="slug", + name="slug", + location="query", + required=False, + ), + Parameter( + argument_name="version", + name="version", + location="query", + required=False, + ), + Parameter( + argument_name="environment", + name="environment", + location="query", + required=False, + ), + Parameter( + argument_name="org_name", + name="org_name", + location="query", + required=False, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.SAFE_READ, +) + + +GET_PROMPT_ID = Operation( + operation_id="getPromptId", + method="GET", + path="/v1/prompt/{prompt_id}", + parameters=( + Parameter( + argument_name="prompt_id", + name="prompt_id", + location="path", + required=True, + ), + Parameter( + argument_name="version", + name="version", + location="query", + required=False, + ), + Parameter( + argument_name="environment", + name="environment", + location="query", + required=False, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.SAFE_READ, +) + + +PATCH_PROMPT_ID = Operation( + operation_id="patchPromptId", + method="PATCH", + path="/v1/prompt/{prompt_id}", + parameters=( + Parameter( + argument_name="prompt_id", + name="prompt_id", + location="path", + required=True, + ), + ), + has_request_body=True, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.NONE, +) + + +DELETE_PROMPT_ID = Operation( + operation_id="deletePromptId", + method="DELETE", + path="/v1/prompt/{prompt_id}", + parameters=( + Parameter( + argument_name="prompt_id", + name="prompt_id", + location="path", + required=True, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.NONE, +) + + +OPERATIONS = { + "postPrompt": POST_PROMPT, + "putPrompt": PUT_PROMPT, + "getPrompt": GET_PROMPT, + "getPromptId": GET_PROMPT_ID, + "patchPromptId": PATCH_PROMPT_ID, + "deletePromptId": DELETE_PROMPT_ID, +} + + +class PromptsAPI(ResourceAPI): + """Generated Prompts REST API.""" + + def post_prompt( + self, + *, + body: "CreatePrompt", + ) -> "Prompt": + return cast( + "Prompt", + self.execute( + POST_PROMPT, + body=body, + ), + ) + + def put_prompt( + self, + *, + body: "CreatePrompt", + ) -> "Prompt": + return cast( + "Prompt", + self.execute( + PUT_PROMPT, + body=body, + ), + ) + + def get_prompt( + self, + *, + limit: "AppLimitParam | None" = None, + starting_after: "StartingAfter | None" = None, + ending_before: "EndingBefore | None" = None, + ids: "Ids | None" = None, + prompt_name: "PromptName | None" = None, + project_name: "ProjectName | None" = None, + project_id: "ProjectIdQuery | None" = None, + slug: "Slug | None" = None, + version: "PromptVersion | None" = None, + environment: "PromptEnvironment | None" = None, + org_name: "OrgName | None" = None, + ) -> "GetPromptResponse": + return cast( + "GetPromptResponse", + self.execute( + GET_PROMPT, + query_parameters={ + "limit": limit, + "starting_after": starting_after, + "ending_before": ending_before, + "ids": ids, + "prompt_name": prompt_name, + "project_name": project_name, + "project_id": project_id, + "slug": slug, + "version": version, + "environment": environment, + "org_name": org_name, + }, + ), + ) + + def get_prompt_id( + self, + prompt_id: "PromptIdParam", + *, + version: "PromptVersion | None" = None, + environment: "PromptEnvironment | None" = None, + ) -> "Prompt": + return cast( + "Prompt", + self.execute( + GET_PROMPT_ID, + path_parameters={"prompt_id": prompt_id}, + query_parameters={"version": version, "environment": environment}, + ), + ) + + def patch_prompt_id( + self, + prompt_id: "PromptIdParam", + *, + body: "PatchPrompt | None" = None, + ) -> "Prompt": + return cast( + "Prompt", + self.execute( + PATCH_PROMPT_ID, + path_parameters={"prompt_id": prompt_id}, + body=body, + ), + ) + + def delete_prompt_id( + self, + prompt_id: "PromptIdParam", + ) -> "Prompt": + return cast( + "Prompt", + self.execute( + DELETE_PROMPT_ID, + path_parameters={"prompt_id": prompt_id}, + ), + ) diff --git a/py/src/braintrust/api/cassettes/test_high_level_load_prompt_uses_generated_resources.yaml b/py/src/braintrust/api/cassettes/test_high_level_load_prompt_uses_generated_resources.yaml new file mode 100644 index 00000000..52178ccf --- /dev/null +++ b/py/src/braintrust/api/cassettes/test_high_level_load_prompt_uses_generated_resources.yaml @@ -0,0 +1,510 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"Braintrust + SDKs","api_url":"https://api.braintrust.dev","git_metadata":{"collect":"some","fields":["commit","branch","tag","dirty","author_name","author_email","commit_message","commit_time"]},"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '376' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-NzM1NDMwMWUtZDI2YS00OTdlLWI4NjktZTg0NTlmZDAyMjJh'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:23 GMT + Etag: + - '"13vsc5ye8flag"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Bt-Was-Udf-Cached: + - 'true' + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - NzM1NDMwMWUtZDI2YS00OTdlLWI4NjktZTg0NTlmZDAyMjJh + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::gqw2f-1788452182643-ffc996c1c907 + status: + code: 200 + message: OK +- request: + body: '{"name": "python-sdk-high-level-prompts-vcr", "org_name": "Braintrust SDKs"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '76' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/project + response: + body: + string: '{"id":"35678b82-85af-4cfe-be4e-a54d23777c2b","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-high-level-prompts-vcr","description":null,"created":"2026-09-03T16:16:24.769Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + Cache-Control: + - no-store, no-cache, must-revalidate, proxy-revalidate + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:25 GMT + ETag: + - W/"113-rcwHcmM04rQSX4ehMbJhZ5ulU6k" + Expires: + - '0' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains + Surrogate-Control: + - no-store + Transfer-Encoding: + - chunked + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 e6bfe249d47d39a52673337cf444c9ce.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - zdHdhs9oEqmxpkRwgMsMTFf584fCdzqD_U4qhi5NIkhQsCQabQHjdA== + X-Amz-Cf-Pop: + - YTO50-P2 + X-Cache: + - Miss from cloudfront + content-length: + - '275' + status: + code: 200 + message: OK +- request: + body: '{"project_id": "35678b82-85af-4cfe-be4e-a54d23777c2b", "name": "Generated + prompts API", "slug": "generated-prompts-api", "description": "loaded through + braintrust.load_prompt", "prompt_data": {"prompt": {"type": "chat", "messages": + [{"role": "user", "content": "Hello {{name}}"}]}, "options": {"model": "gpt-5-mini"}}, + "tags": ["python-sdk-vcr"]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '346' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/prompt + response: + body: + string: '{"log_id":"p","project_id":"35678b82-85af-4cfe-be4e-a54d23777c2b","slug":"generated-prompts-api","name":"Generated + prompts API","description":"loaded through braintrust.load_prompt","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Hello + {{name}}"}]},"options":{"model":"gpt-5-mini"}},"tags":["python-sdk-vcr"],"id":"ce6dc63d-3e88-4909-8d41-93de0161ffc2","created":"2026-09-03T16:16:25.473Z","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","_xact_id":"1000197800255422556"}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + Cache-Control: + - no-store, no-cache, must-revalidate, proxy-revalidate + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:26 GMT + ETag: + - W/"1f4-khFsYy3ummb/hydHGyN7R1VTYXc" + Expires: + - '0' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains + Surrogate-Control: + - no-store + Transfer-Encoding: + - chunked + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 12aa3fefbdb5e80269e58f34f94a99e8.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - IUmtSayH5gFASNgbCTuzsFagVxFtZ5hlUt3aRgJvvQT5ZtsXqIn9UQ== + X-Amz-Cf-Pop: + - YTO50-P2 + X-Cache: + - Miss from cloudfront + content-length: + - '500' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"Braintrust + SDKs","api_url":"https://api.braintrust.dev","git_metadata":{"collect":"some","fields":["commit","branch","tag","dirty","author_name","author_email","commit_message","commit_time"]},"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '376' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-OWFjZDVjMGEtN2JlYi00OTZmLTk5MmQtNTc4ZDI3ODg1Yjgx'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:26 GMT + Etag: + - '"13vsc5ye8flag"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Bt-Was-Udf-Cached: + - 'true' + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - OWFjZDVjMGEtN2JlYi00OTZmLTk5MmQtNTc4ZDI3ODg1Yjgx + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::7mlpr-1788452186280-86f358a0c51a + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/v1/prompt?project_id=35678b82-85af-4cfe-be4e-a54d23777c2b&slug=generated-prompts-api&version=1000197800255422556 + response: + body: + string: '{"objects":[{"_xact_id":"1000197800255422556","created":"2026-09-03T16:16:25.473Z","description":"loaded + through braintrust.load_prompt","function_type":null,"id":"ce6dc63d-3e88-4909-8d41-93de0161ffc2","log_id":"p","metadata":null,"name":"Generated + prompts API","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","project_id":"35678b82-85af-4cfe-be4e-a54d23777c2b","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Hello + {{name}}"}]},"options":{"model":"gpt-5-mini"}},"slug":"generated-prompts-api","tags":["python-sdk-vcr"],"function_data":{"type":"prompt"}}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + Cache-Control: + - no-store, no-cache, must-revalidate, proxy-revalidate + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:27 GMT + ETag: + - W/"249-/cP2GbxxnMYMEzi0cdC3Il6mOGw" + Expires: + - '0' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains + Surrogate-Control: + - no-store + Transfer-Encoding: + - chunked + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 10f12ad63ad88e4e38e4e73deb3e9570.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - 52-r9sYrB8-hMpL--AwgR9pW1HbnC2V_hAiQpp5cR8kC4mJneZEhLg== + X-Amz-Cf-Pop: + - YTO50-P2 + X-Cache: + - Miss from cloudfront + content-length: + - '585' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/v1/prompt/ce6dc63d-3e88-4909-8d41-93de0161ffc2 + response: + body: + string: '{"_xact_id":"1000197800255422556","created":"2026-09-03T16:16:25.473Z","description":"loaded + through braintrust.load_prompt","function_type":null,"id":"ce6dc63d-3e88-4909-8d41-93de0161ffc2","log_id":"p","metadata":null,"name":"Generated + prompts API","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","project_id":"35678b82-85af-4cfe-be4e-a54d23777c2b","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Hello + {{name}}"}]},"options":{"model":"gpt-5-mini"}},"slug":"generated-prompts-api","tags":["python-sdk-vcr"],"function_data":{"type":"prompt"}}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + Cache-Control: + - no-store, no-cache, must-revalidate, proxy-revalidate + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:28 GMT + ETag: + - W/"23b-/KFI4pcarF3TZSFsVQiLr5y6IQI" + Expires: + - '0' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains + Surrogate-Control: + - no-store + Transfer-Encoding: + - chunked + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 53d47b61433f6e1682b806fc166731be.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - sjzu0I1pZ3QS-Oyz0oJWrf0m6MrDWvBs5fkyaDCPsdJxMpwgXl7RcA== + X-Amz-Cf-Pop: + - YTO50-P2 + X-Cache: + - Miss from cloudfront + content-length: + - '571' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: DELETE + uri: https://api.braintrust.dev/v1/prompt/ce6dc63d-3e88-4909-8d41-93de0161ffc2 + response: + body: + string: '{"log_id":"p","project_id":"35678b82-85af-4cfe-be4e-a54d23777c2b","slug":"generated-prompts-api","id":"ce6dc63d-3e88-4909-8d41-93de0161ffc2","created":"2026-09-03T16:16:29.251Z","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","_object_delete":true,"_xact_id":"1000197800255620906"}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + Cache-Control: + - no-store, no-cache, must-revalidate, proxy-revalidate + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:29 GMT + ETag: + - W/"119-eTVpPLlxCg6iQAgsHfDOp+60+8w" + Expires: + - '0' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains + Surrogate-Control: + - no-store + Transfer-Encoding: + - chunked + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 04fa8a9e73b27e301fb4b6d36f313186.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - lja73dv5pD7y5H60Cv5NAQMhj_ffTFgG-O3GKHeJhSzGJ0pCBX6QWw== + X-Amz-Cf-Pop: + - YTO50-P2 + X-Cache: + - Miss from cloudfront + content-length: + - '281' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: DELETE + uri: https://api.braintrust.dev/v1/project/35678b82-85af-4cfe-be4e-a54d23777c2b + response: + body: + string: '{"id":"35678b82-85af-4cfe-be4e-a54d23777c2b","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-high-level-prompts-vcr","description":null,"created":"2026-09-03T16:16:24.769Z","deleted_at":"2026-09-03T16:16:29.511Z","user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + Cache-Control: + - no-store, no-cache, must-revalidate, proxy-revalidate + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:29 GMT + ETag: + - W/"129-1akA1b547h8pAi2rMOvr8w8D+nw" + Expires: + - '0' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains + Surrogate-Control: + - no-store + Transfer-Encoding: + - chunked + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 777f4a7ed43b40353f84311869e119c8.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - p_VxpYJf8HvDK0-wxytTarngh8wOG30YrZ_cQAXZydTw3zRxzu8f1g== + X-Amz-Cf-Pop: + - YTO50-P2 + X-Cache: + - Miss from cloudfront + content-length: + - '297' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/api/cassettes/test_prompts_end_to_end_with_real_backend.yaml b/py/src/braintrust/api/cassettes/test_prompts_end_to_end_with_real_backend.yaml new file mode 100644 index 00000000..e97dee89 --- /dev/null +++ b/py/src/braintrust/api/cassettes/test_prompts_end_to_end_with_real_backend.yaml @@ -0,0 +1,552 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"Braintrust + SDKs","api_url":"https://api.braintrust.dev","git_metadata":{"collect":"some","fields":["commit","branch","tag","dirty","author_name","author_email","commit_message","commit_time"]},"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '376' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ZTMyZGU1OTUtOTBiYi00NTA2LWEyNDEtMDk4ZmQ5MjE4NDVl'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:14 GMT + Etag: + - '"13vsc5ye8flag"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - ZTMyZGU1OTUtOTBiYi00NTA2LWEyNDEtMDk4ZmQ5MjE4NDVl + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::vzb2v-1788452174617-6539d4f5645d + status: + code: 200 + message: OK +- request: + body: '{"name": "python-sdk-generated-prompts-vcr", "org_name": "Braintrust SDKs"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '75' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/project + response: + body: + string: '{"id":"278bef0f-7815-4f7f-804e-7ed91f30f9f5","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-generated-prompts-vcr","description":null,"created":"2026-09-03T16:16:15.478Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + Cache-Control: + - no-store, no-cache, must-revalidate, proxy-revalidate + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:16 GMT + ETag: + - W/"112-GJP20bI35jxPWLmr5GO0u0HVWR4" + Expires: + - '0' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains + Surrogate-Control: + - no-store + Transfer-Encoding: + - chunked + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 a7af18c87ffc07d74544efce5f2b0f9c.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - ch52YzDpmcBPkNoxHu6sSTJDXjmmllwyR65sFi7gU-TB7TzWZbMT3A== + X-Amz-Cf-Pop: + - YTO50-P2 + X-Cache: + - Miss from cloudfront + content-length: + - '274' + status: + code: 200 + message: OK +- request: + body: '{"project_id": "278bef0f-7815-4f7f-804e-7ed91f30f9f5", "name": "Generated + prompts API", "slug": "generated-prompts-api", "description": "created by the + Python SDK VCR test", "prompt_data": {"prompt": {"type": "chat", "messages": + [{"role": "user", "content": "Hello {{name}}"}]}, "options": {"model": "gpt-5-mini"}}, + "tags": ["python-sdk-vcr"]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '343' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/prompt + response: + body: + string: '{"log_id":"p","project_id":"278bef0f-7815-4f7f-804e-7ed91f30f9f5","slug":"generated-prompts-api","name":"Generated + prompts API","description":"created by the Python SDK VCR test","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Hello + {{name}}"}]},"options":{"model":"gpt-5-mini"}},"tags":["python-sdk-vcr"],"id":"cff6c2ab-4828-47d4-ba6a-5c866d2016dd","created":"2026-09-03T16:16:16.158Z","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","_xact_id":"1000197800254768533"}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + Cache-Control: + - no-store, no-cache, must-revalidate, proxy-revalidate + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:16 GMT + ETag: + - W/"1f1-iplTYxJifPFUjPGGDQWLAyMGZi0" + Expires: + - '0' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains + Surrogate-Control: + - no-store + Transfer-Encoding: + - chunked + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 6889869bf680fe34cca722f0a05e1106.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - c4xepx3zhu8XQM7yTJnVlTjMDbbe5CcwWaMfERvS8AF0jpl7hqjm3g== + X-Amz-Cf-Pop: + - YTO50-P2 + X-Cache: + - Miss from cloudfront + content-length: + - '497' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/v1/prompt?limit=1&project_id=278bef0f-7815-4f7f-804e-7ed91f30f9f5&slug=generated-prompts-api + response: + body: + string: '{"objects":[{"_xact_id":"1000197800254768533","created":"2026-09-03T16:16:16.158Z","description":"created + by the Python SDK VCR test","function_type":null,"id":"cff6c2ab-4828-47d4-ba6a-5c866d2016dd","log_id":"p","metadata":null,"name":"Generated + prompts API","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","project_id":"278bef0f-7815-4f7f-804e-7ed91f30f9f5","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Hello + {{name}}"}]},"options":{"model":"gpt-5-mini"}},"slug":"generated-prompts-api","tags":["python-sdk-vcr"],"function_data":{"type":"prompt"}}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + Cache-Control: + - no-store, no-cache, must-revalidate, proxy-revalidate + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:18 GMT + ETag: + - W/"246-YYInyIiiC6KCQJDWifxh02NCe5Q" + Expires: + - '0' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains + Surrogate-Control: + - no-store + Transfer-Encoding: + - chunked + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 cdd327922be1fd75b18f2ae0982269cc.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - EQ--tw0iizL6JFB_CiiRuZa27eKCKl0nx52KMnXb_i0EbBa8tf4RAw== + X-Amz-Cf-Pop: + - YTO50-P2 + X-Cache: + - Miss from cloudfront + content-length: + - '582' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/v1/prompt/cff6c2ab-4828-47d4-ba6a-5c866d2016dd?version=1000197800254768533 + response: + body: + string: '{"_xact_id":"1000197800254768533","created":"2026-09-03T16:16:16.158Z","description":"created + by the Python SDK VCR test","function_type":null,"id":"cff6c2ab-4828-47d4-ba6a-5c866d2016dd","log_id":"p","metadata":null,"name":"Generated + prompts API","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","project_id":"278bef0f-7815-4f7f-804e-7ed91f30f9f5","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Hello + {{name}}"}]},"options":{"model":"gpt-5-mini"}},"slug":"generated-prompts-api","tags":["python-sdk-vcr"],"function_data":{"type":"prompt"}}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + Cache-Control: + - no-store, no-cache, must-revalidate, proxy-revalidate + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:19 GMT + ETag: + - W/"238-TTpJaVR8yJjgVfsigMBRd+zGZcM" + Expires: + - '0' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains + Surrogate-Control: + - no-store + Transfer-Encoding: + - chunked + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 7293b56f3a0eb541aadcbcaa0146d528.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - v3TG1SjDO0JN0dsEWGR6t-uKwFT2JGvH6JPxuI-3ysWD0F7J0FWviA== + X-Amz-Cf-Pop: + - YTO50-P2 + X-Cache: + - Miss from cloudfront + content-length: + - '568' + status: + code: 200 + message: OK +- request: + body: '{"description": "updated by the Python SDK VCR test"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '53' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: PATCH + uri: https://api.braintrust.dev/v1/prompt/cff6c2ab-4828-47d4-ba6a-5c866d2016dd + response: + body: + string: '{"name":"Generated prompts API","slug":"generated-prompts-api","tags":["python-sdk-vcr"],"log_id":"p","project_id":"278bef0f-7815-4f7f-804e-7ed91f30f9f5","description":"updated + by the Python SDK VCR test","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Hello + {{name}}"}]},"options":{"model":"gpt-5-mini"}},"id":"cff6c2ab-4828-47d4-ba6a-5c866d2016dd","created":"2026-09-03T16:16:16.158Z","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","_xact_id":"1000197800255029966"}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + Cache-Control: + - no-store, no-cache, must-revalidate, proxy-revalidate + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:20 GMT + ETag: + - W/"1f1-agESJRSEREdJ0yKWri4O3BlqHJE" + Expires: + - '0' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains + Surrogate-Control: + - no-store + Transfer-Encoding: + - chunked + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 cb0c6226aa19d81a39519501df383968.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - Gtn6lhXQw3eOnv_f0-HdRtmZlDa3pAhKdIoxOijVZewwIHAbNACHXg== + X-Amz-Cf-Pop: + - YTO50-P2 + X-Cache: + - Miss from cloudfront + content-length: + - '497' + status: + code: 200 + message: OK +- request: + body: '{"project_id": "278bef0f-7815-4f7f-804e-7ed91f30f9f5", "name": "Generated + prompts API", "slug": "generated-prompts-api", "description": "replaced by the + Python SDK VCR test", "prompt_data": {"prompt": {"type": "chat", "messages": + [{"role": "user", "content": "Hello {{name}}"}]}, "options": {"model": "gpt-5-mini"}}, + "tags": ["python-sdk-vcr"]}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '344' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: PUT + uri: https://api.braintrust.dev/v1/prompt + response: + body: + string: '{"log_id":"p","project_id":"278bef0f-7815-4f7f-804e-7ed91f30f9f5","slug":"generated-prompts-api","name":"Generated + prompts API","description":"replaced by the Python SDK VCR test","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Hello + {{name}}"}]},"options":{"model":"gpt-5-mini"}},"tags":["python-sdk-vcr"],"id":"cff6c2ab-4828-47d4-ba6a-5c866d2016dd","created":"2026-09-03T16:16:20.784Z","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","_xact_id":"1000197800255035082"}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + Cache-Control: + - no-store, no-cache, must-revalidate, proxy-revalidate + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:20 GMT + ETag: + - W/"1f2-MA8vxywhwutUixuR5QUqcLXnlE0" + Expires: + - '0' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains + Surrogate-Control: + - no-store + Transfer-Encoding: + - chunked + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 39d0b6c3836d173e719889fc86d67ce8.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - kTnJ1lmhTiD-OyCd-3q8_vKv0mBsN4iOny2vzYEvGcZLlQ7QE0LqdQ== + X-Amz-Cf-Pop: + - YTO50-P2 + X-Cache: + - Miss from cloudfront + content-length: + - '498' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: DELETE + uri: https://api.braintrust.dev/v1/prompt/cff6c2ab-4828-47d4-ba6a-5c866d2016dd + response: + body: + string: '{"log_id":"p","project_id":"278bef0f-7815-4f7f-804e-7ed91f30f9f5","slug":"generated-prompts-api","id":"cff6c2ab-4828-47d4-ba6a-5c866d2016dd","created":"2026-09-03T16:16:21.456Z","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","_object_delete":true,"_xact_id":"1000197800255098093"}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + Cache-Control: + - no-store, no-cache, must-revalidate, proxy-revalidate + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:21 GMT + ETag: + - W/"119-zIgmftUahzqVSbz5pBENBaC0pjM" + Expires: + - '0' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains + Surrogate-Control: + - no-store + Transfer-Encoding: + - chunked + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 67dd4d73b80aece69a8e725c6d612b6e.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - itFJkntFPdlggrBUxxZKlLrKVMav46wvno1ec7I4sgKM9vCE5mflnA== + X-Amz-Cf-Pop: + - YTO50-P2 + X-Cache: + - Miss from cloudfront + content-length: + - '281' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: DELETE + uri: https://api.braintrust.dev/v1/project/278bef0f-7815-4f7f-804e-7ed91f30f9f5 + response: + body: + string: '{"id":"278bef0f-7815-4f7f-804e-7ed91f30f9f5","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-generated-prompts-vcr","description":null,"created":"2026-09-03T16:16:15.478Z","deleted_at":"2026-09-03T16:16:22.059Z","user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + Cache-Control: + - no-store, no-cache, must-revalidate, proxy-revalidate + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 03 Sep 2026 16:16:22 GMT + ETag: + - W/"128-IPMOAf47NX3opgJHYOPioAnSS6U" + Expires: + - '0' + Strict-Transport-Security: + - max-age=31536000; includeSubDomains + Surrogate-Control: + - no-store + Transfer-Encoding: + - chunked + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 3340b5a392e45fce453c4d978abfd6be.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - IKYyqnv_fYE3nsmOg0tEZToXlmz8mdRLl2jNkPr2IeiBMUBzeSIWzA== + X-Amz-Cf-Pop: + - YTO50-P2 + X-Cache: + - Miss from cloudfront + content-length: + - '296' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/api/client.py b/py/src/braintrust/api/client.py index 6a6e00e2..0bce27d4 100644 --- a/py/src/braintrust/api/client.py +++ b/py/src/braintrust/api/client.py @@ -157,11 +157,13 @@ def _initialize_services(self, api_key: str) -> None: from ._generated.datasets import DatasetsAPI from ._generated.experiments import ExperimentsAPI from ._generated.projects import ProjectsAPI + from ._generated.prompts import PromptsAPI self.api_key = api_key self.datasets = DatasetsAPI(self.transport, self.router, api_key) self.experiments = ExperimentsAPI(self.transport, self.router, api_key) self.projects = ProjectsAPI(self.transport, self.router, api_key) + self.prompts = PromptsAPI(self.transport, self.router, api_key) def close(self) -> None: """Close the transport when it was created by this client.""" diff --git a/py/src/braintrust/api/test_generated_models.py b/py/src/braintrust/api/test_generated_models.py index 4b3562ac..afc6f778 100644 --- a/py/src/braintrust/api/test_generated_models.py +++ b/py/src/braintrust/api/test_generated_models.py @@ -23,16 +23,20 @@ def test_generated_models_import_on_supported_python(): from braintrust.api._generated import experiments as experiment_bindings from braintrust.api._generated import models from braintrust.api._generated import projects as project_bindings + from braintrust.api._generated import prompts as prompt_bindings assert is_typeddict(models.Dataset) assert is_typeddict(models.Experiment) assert is_typeddict(models.Project) + assert is_typeddict(models.Prompt) assert models.DatasetIdParam is str assert models.ExperimentIdParam is str assert models.ProjectIdParam is str + assert models.PromptIdParam is str assert get_type_hints(dataset_bindings.DatasetsAPI.get_dataset)["return"] is models.GetDatasetResponse assert get_type_hints(experiment_bindings.ExperimentsAPI.get_experiment)["return"] is models.GetExperimentResponse assert get_type_hints(project_bindings.ProjectsAPI.get_project)["return"] is models.GetProjectResponse + assert get_type_hints(prompt_bindings.PromptsAPI.get_prompt)["return"] is models.GetPromptResponse def test_generated_package_content_is_installed(): @@ -44,9 +48,11 @@ def test_generated_package_content_is_installed(): assert generated.joinpath("models", "datasets.py").is_file() assert generated.joinpath("models", "experiments.py").is_file() assert generated.joinpath("models", "projects.py").is_file() + assert generated.joinpath("models", "prompts.py").is_file() assert generated.joinpath("datasets.py").is_file() assert generated.joinpath("experiments.py").is_file() assert generated.joinpath("projects.py").is_file() + assert generated.joinpath("prompts.py").is_file() def test_rest_and_logging_type_surfaces_have_reviewed_overlap(): @@ -55,4 +61,4 @@ def test_rest_and_logging_type_surfaces_have_reviewed_overlap(): overlap = set(generated_types.__all__) & set(types.__all__) - assert overlap == {"Dataset", "Experiment", "Project"} + assert overlap == {"Dataset", "Experiment", "Project", "Prompt"} diff --git a/py/src/braintrust/api/test_prompts.py b/py/src/braintrust/api/test_prompts.py new file mode 100644 index 00000000..9c63fe23 --- /dev/null +++ b/py/src/braintrust/api/test_prompts.py @@ -0,0 +1,152 @@ +import contextlib + +import braintrust +import pytest +from braintrust.api import BraintrustClient, BraintrustOpenApiClient +from braintrust.api._generated.prompts import OPERATIONS +from braintrust.api._test_server import scripted_server +from braintrust.api.policies import RetryMode + + +def _create_prompt_body(project_id: str, description: str) -> dict: + return { + "project_id": project_id, + "name": "Generated prompts API", + "slug": "generated-prompts-api", + "description": description, + "prompt_data": { + "prompt": { + "type": "chat", + "messages": [{"role": "user", "content": "Hello {{name}}"}], + }, + "options": {"model": "gpt-5-mini"}, + }, + "tags": ["python-sdk-vcr"], + } + + +def test_all_prompt_operations_have_complete_retry_classification(): + assert {name: operation.retry_mode for name, operation in OPERATIONS.items()} == { + "postPrompt": RetryMode.NONE, + "putPrompt": RetryMode.NONE, + "getPrompt": RetryMode.SAFE_READ, + "getPromptId": RetryMode.SAFE_READ, + "patchPromptId": RetryMode.NONE, + "deletePromptId": RetryMode.NONE, + } + + +def test_get_prompt_preserves_exact_wire_query_and_additive_response_fields(): + response = b'{"objects":[{"id":"prompt-id","future_field":{"preserved":true}}]}' + with scripted_server([(200, {"Content-Type": "application/json"}, response)]) as (api_url, handler): + with BraintrustOpenApiClient(api_key="test-key", api_url=api_url) as client: + prompts = client.prompts.get_prompt( + limit=1, + project_name="test project", + slug="prompt/slug", + version="123", + ) + + assert handler.requests[0][:2] == ( + "GET", + "/v1/prompt?limit=1&project_name=test%20project&slug=prompt%2Fslug&version=123", + ) + assert prompts["objects"][0]["future_field"] == {"preserved": True} + + +@pytest.mark.vcr +def test_prompts_end_to_end_with_real_backend(api_key): + project_name = "python-sdk-generated-prompts-vcr" + cleanup_project_id = None + cleanup_prompt_id = None + + with BraintrustClient(api_key=api_key) as client: + try: + discovery = client.auth.login() + project = client.openapi.projects.post_project( + body={"name": project_name, "org_name": discovery.organization.name} + ) + cleanup_project_id = project["id"] + created = client.openapi.prompts.post_prompt( + body=_create_prompt_body( + project["id"], + description="created by the Python SDK VCR test", + ) + ) + cleanup_prompt_id = created["id"] + listed = client.openapi.prompts.get_prompt( + project_id=project["id"], + slug=created["slug"], + limit=1, + ) + fetched = client.openapi.prompts.get_prompt_id(created["id"], version=created["_xact_id"]) + updated = client.openapi.prompts.patch_prompt_id( + created["id"], body={"description": "updated by the Python SDK VCR test"} + ) + replaced = client.openapi.prompts.put_prompt( + body=_create_prompt_body( + project["id"], + description="replaced by the Python SDK VCR test", + ) + ) + deleted = client.openapi.prompts.delete_prompt_id(created["id"]) + cleanup_prompt_id = None + client.openapi.projects.delete_project_id(project["id"]) + cleanup_project_id = None + finally: + if cleanup_prompt_id is not None: + with contextlib.suppress(Exception): + client.openapi.prompts.delete_prompt_id(cleanup_prompt_id) + if cleanup_project_id is not None: + with contextlib.suppress(Exception): + client.openapi.projects.delete_project_id(cleanup_project_id) + + assert created["slug"] == "generated-prompts-api" + assert [prompt["id"] for prompt in listed["objects"]] == [created["id"]] + assert fetched["id"] == created["id"] + assert updated["description"] == "updated by the Python SDK VCR test" + assert replaced["id"] == created["id"] + assert replaced["description"] == "replaced by the Python SDK VCR test" + assert deleted["id"] == created["id"] + + +@pytest.mark.vcr +def test_high_level_load_prompt_uses_generated_resources(api_key): + project_name = "python-sdk-high-level-prompts-vcr" + cleanup_project_id = None + cleanup_prompt_id = None + + with BraintrustClient(api_key=api_key) as client: + try: + discovery = client.auth.login() + project = client.openapi.projects.post_project( + body={"name": project_name, "org_name": discovery.organization.name} + ) + cleanup_project_id = project["id"] + created = client.openapi.prompts.post_prompt( + body=_create_prompt_body(project["id"], description="loaded through braintrust.load_prompt") + ) + cleanup_prompt_id = created["id"] + + by_slug = braintrust.load_prompt( + project_id=project["id"], + slug=created["slug"], + version=created["_xact_id"], + api_key=api_key, + org_name=discovery.organization.name, + ) + by_id = braintrust.load_prompt( + id=created["id"], + api_key=api_key, + org_name=discovery.organization.name, + ) + + assert by_slug.build(name="Ada")["messages"][0]["content"] == "Hello Ada" + assert by_id.id == created["id"] + finally: + if cleanup_prompt_id is not None: + with contextlib.suppress(Exception): + client.openapi.prompts.delete_prompt_id(cleanup_prompt_id) + if cleanup_project_id is not None: + with contextlib.suppress(Exception): + client.openapi.projects.delete_project_id(cleanup_project_id) diff --git a/py/src/braintrust/api/types/__init__.py b/py/src/braintrust/api/types/__init__.py index fdacfe8b..f12b1066 100644 --- a/py/src/braintrust/api/types/__init__.py +++ b/py/src/braintrust/api/types/__init__.py @@ -4,6 +4,7 @@ CreateDataset, CreateExperiment, CreateProject, + CreatePrompt, Dataset, Experiment, FeedbackDatasetEventRequest, @@ -15,13 +16,16 @@ GetDatasetResponse, GetExperimentResponse, GetProjectResponse, + GetPromptResponse, InsertDatasetEventRequest, InsertEventsResponse, InsertExperimentEventRequest, PatchDataset, PatchExperiment, PatchProject, + PatchPrompt, Project, + Prompt, SummarizeDatasetResponse, SummarizeExperimentResponse, ) @@ -31,6 +35,7 @@ "CreateDataset", "CreateExperiment", "CreateProject", + "CreatePrompt", "Dataset", "Experiment", "FeedbackDatasetEventRequest", @@ -42,13 +47,16 @@ "GetDatasetResponse", "GetExperimentResponse", "GetProjectResponse", + "GetPromptResponse", "InsertDatasetEventRequest", "InsertEventsResponse", "InsertExperimentEventRequest", "PatchDataset", "PatchExperiment", "PatchProject", + "PatchPrompt", "Project", + "Prompt", "SummarizeDatasetResponse", "SummarizeExperimentResponse", ] diff --git a/py/src/braintrust/logger.py b/py/src/braintrust/logger.py index 70cf41b6..24dd1e20 100644 --- a/py/src/braintrust/logger.py +++ b/py/src/braintrust/logger.py @@ -451,7 +451,10 @@ class _LoaderLoginOptions: cache_namespace: str -class _LoaderLoginEntry: +_LoaderResource = TypeVar("_LoaderResource", HTTPConnection, BraintrustClient) + + +class _LoaderLoginEntry(Generic[_LoaderResource]): """One credential's loader login, tracking who is responsible for closing it. The login runs outside the cache lock, so an entry can be evicted while its @@ -461,13 +464,13 @@ class _LoaderLoginEntry: connection that another caller is still issuing a request on. """ - def __init__(self, factory: Callable[[], HTTPConnection]): - self._lazy: LazyValue[HTTPConnection] = LazyValue(factory, use_mutex=True) + def __init__(self, factory: Callable[[], _LoaderResource]): + self._lazy: LazyValue[_LoaderResource] = LazyValue(factory, use_mutex=True) self._lock = threading.Lock() self._evicted = False self._active = 0 - def acquire(self) -> HTTPConnection: + def acquire(self) -> _LoaderResource: with self._lock: self._active += 1 try: @@ -499,7 +502,11 @@ def _close(self) -> None: class BraintrustState: def __init__(self): self.id = str(uuid.uuid4()) - self._loader_login_cache: LRUCache[str, _LoaderLoginEntry] = LRUCache( + self._loader_login_cache: LRUCache[str, _LoaderLoginEntry[HTTPConnection]] = LRUCache( + max_size=16, + on_remove=self._evict_loader_login_entry, + ) + self._loader_api_client_cache: LRUCache[str, _LoaderLoginEntry[BraintrustClient]] = LRUCache( max_size=16, on_remove=self._evict_loader_login_entry, ) @@ -585,6 +592,7 @@ def default_get_api_conn(): def reset_login_info(self): self._loader_login_cache.clear() + self._loader_api_client_cache.clear() self.app_url: str | None = None self.app_public_url: str | None = None @@ -604,7 +612,7 @@ def reset_login_info(self): self._user_info: Mapping[str, Any] | None = None @staticmethod - def _evict_loader_login_entry(_key: str, entry: "_LoaderLoginEntry") -> None: + def _evict_loader_login_entry(_key: str, entry: "_LoaderLoginEntry[_LoaderResource]") -> None: entry.evict() def reset_parent_state(self): @@ -666,6 +674,7 @@ async def flush_otel(self) -> None: def copy_state(self, other: "BraintrustState"): """Copy login information from another BraintrustState instance.""" self._loader_login_cache.clear() + self._loader_api_client_cache.clear() self.__dict__.update( { k: v @@ -684,6 +693,7 @@ def copy_state(self, other: "BraintrustState"): "_context_manager_lock", "_client_lock", "_loader_login_cache", + "_loader_api_client_cache", ) } ) @@ -759,37 +769,64 @@ def user_info(self) -> Mapping[str, Any]: self._user_info = self.api_conn().get_json("ping") return self._user_info - @contextlib.contextmanager - def loader_conn(self, options: "_LoaderLoginOptions") -> "Iterator[HTTPConnection]": - """Yield the API connection for one loader call, releasing it on exit. - - The global login's connection is shared and outlives the call, so it is - yielded as-is. A per-credential connection is owned by its cache entry, - which needs the release to know when an evicted one is safe to close. - """ - - if ( + def _uses_active_loader_login(self, options: "_LoaderLoginOptions") -> bool: + return ( self.logged_in and self.login_token == options.api_key and self.app_url == options.app_url and (options.org_name is None or self.org_name == options.org_name) - ): - yield self.api_conn() - return + ) + @contextlib.contextmanager + def _cached_loader_resource( + self, + cache: "LRUCache[str, _LoaderLoginEntry[_LoaderResource]]", + cache_key: str, + factory: "Callable[[], _LoaderResource]", + ) -> "Iterator[_LoaderResource]": with self._client_lock: try: - entry = self._loader_login_cache.get(options.cache_namespace) + entry = cache.get(cache_key) except KeyError: - entry = _LoaderLoginEntry(lambda: _login_loader_conn(options)) - self._loader_login_cache.set(options.cache_namespace, entry) + entry = _LoaderLoginEntry(factory) + cache.set(cache_key, entry) - conn = entry.acquire() + resource = entry.acquire() try: - yield conn + yield resource finally: entry.release() + @contextlib.contextmanager + def loader_conn(self, options: "_LoaderLoginOptions") -> "Iterator[HTTPConnection]": + """Yield the API connection for one loader call, releasing it on exit.""" + + if self._uses_active_loader_login(options): + yield self.api_conn() + return + + with self._cached_loader_resource( + self._loader_login_cache, + options.cache_namespace, + lambda: _login_loader_conn(options), + ) as conn: + yield conn + + @contextlib.contextmanager + def loader_api_client(self, options: "_LoaderLoginOptions") -> "Iterator[BraintrustOpenApiClient]": + """Yield the generated API client for one loader call, releasing it on exit.""" + + if self._uses_active_loader_login(options): + yield self.api_client() + return + + with self._cached_loader_resource( + self._loader_api_client_cache, + options.cache_namespace, + lambda: _login_loader_client(options), + ) as client: + yield client.openapi + def global_bg_logger(self) -> "_BackgroundLogger": return getattr(self._override_bg_logger, "logger", None) or self._global_bg_logger.get() @@ -2041,24 +2078,21 @@ def compute_metadata(): ) cache_namespace = login_options.cache_namespace try: - with _state.loader_conn(login_options) as conn: + with _state.loader_api_client(login_options) as api_client: if id: - # Load prompt by ID using the /v1/prompt/{id} endpoint - prompt_args = _populate_args({}, version=version, environment=effective_environment) - response = conn.get_json(f"/v1/prompt/{id}", prompt_args) - # Wrap single prompt response in objects array to match list API format - if response is not None: - response = {"objects": [response]} + resp_prompt = api_client.prompts.get_prompt_id( + id, + version=str(version) if version is not None else None, + environment=effective_environment, + ) else: - args = _populate_args( - {}, + response = api_client.prompts.get_prompt( project_name=project, project_id=project_id, slug=slug, - version=version, + version=str(version) if version is not None else None, environment=effective_environment, ) - response = conn.get_json("/v1/prompt", args) except Exception as server_error: if not _is_loader_cache_fallback_error(server_error): raise @@ -2087,19 +2121,17 @@ def compute_metadata(): raise ValueError( f"Prompt {slug} (version {version or 'latest'}) not found in {project or project_id} (not found on server or in local cache): {cache_error}" ) from server_error - if response is None or "objects" not in response or len(response["objects"]) == 0: - if id: + if id: + if resp_prompt is None: raise ValueError(f"Prompt with id {id} not found.") - else: + else: + if response is None or "objects" not in response or len(response["objects"]) == 0: raise ValueError(f"Prompt {slug} not found in project {project or project_id}.") - elif len(response["objects"]) > 1: - if id: - raise ValueError(f"Multiple prompts found with id {id}. This should never happen.") - else: + if len(response["objects"]) > 1: raise ValueError( f"Multiple prompts found with slug {slug} in project {project or project_id}. This should never happen." ) - resp_prompt = response["objects"][0] + resp_prompt = response["objects"][0] prompt = PromptSchema.from_dict_deep(resp_prompt) try: if id: @@ -2392,6 +2424,11 @@ def _authenticated_api_conn(api_url: str, api_key: str) -> HTTPConnection: return conn +def _login_loader_client(options: _LoaderLoginOptions) -> BraintrustClient: + client, _ = _login_with_api_key(app_url=options.app_url, api_key=options.api_key, org_name=options.org_name) + return client + + def _login_loader_conn(options: _LoaderLoginOptions) -> HTTPConnection: client, login_result = _login_with_api_key( app_url=options.app_url, api_key=options.api_key, org_name=options.org_name diff --git a/py/src/braintrust/test_logger.py b/py/src/braintrust/test_logger.py index 772bed16..e41e805a 100644 --- a/py/src/braintrust/test_logger.py +++ b/py/src/braintrust/test_logger.py @@ -98,6 +98,26 @@ def test_loader_request_state_closes_connections_on_eviction_and_reset(): second_conn.close.assert_called_once() +def test_loader_request_state_closes_openapi_clients_on_eviction_and_reset(): + state = BraintrustState() + state._loader_api_client_cache = LRUCache(max_size=1, on_remove=state._evict_loader_login_entry) + first_client = MagicMock() + second_client = MagicMock() + + with patch.object(logger, "_login_loader_client", side_effect=[first_client, second_client]): + with state.loader_api_client(_loader_options("first-api-key", "first")) as api_client: + assert api_client is first_client.openapi + with state.loader_api_client(_loader_options("second-api-key", "second")) as api_client: + assert api_client is second_client.openapi + + first_client.close.assert_called_once() + second_client.close.assert_not_called() + + state.reset_login_info() + + second_client.close.assert_called_once() + + def test_loader_request_state_closes_state_evicted_while_login_is_pending(): state = BraintrustState() state._loader_login_cache = LRUCache(max_size=1, on_remove=state._evict_loader_login_entry) @@ -460,12 +480,12 @@ def test_load_prompt_uses_explicit_api_key_without_changing_global_login(): simulate_login() original_login_token = logger._state.login_token prompt_cache = PromptCache(memory_cache=LRUCache(max_size=10)) - request_conn = MagicMock() - request_conn.get_json.return_value = _prompt_response("saved-prompt") + request_client = MagicMock() + request_client.openapi.prompts.get_prompt.return_value = _prompt_response("saved-prompt") with ( patch.object(logger._state, "_prompt_cache", prompt_cache), - patch.object(logger, "_login_loader_conn", return_value=request_conn) as mock_login_conn, + patch.object(logger, "_login_loader_client", return_value=request_client) as mock_login_client, ): prompt = braintrust.load_prompt( project="test-project", @@ -474,13 +494,31 @@ def test_load_prompt_uses_explicit_api_key_without_changing_global_login(): ) assert prompt.slug == "saved-prompt" - (called_options,) = mock_login_conn.call_args.args + request_client.openapi.prompts.get_prompt.assert_called_once_with( + project_name="test-project", + project_id=None, + slug="saved-prompt", + version=None, + environment=None, + ) + (called_options,) = mock_login_client.call_args.args assert called_options.app_url == logger._state.app_url assert called_options.api_key == "prompt-api-key" assert called_options.org_name is None assert logger._state.login_token == original_login_token +def test_load_prompt_by_id_reports_an_empty_response_as_not_found(): + simulate_login() + mock_api_client = MagicMock() + mock_api_client.prompts.get_prompt_id.return_value = None + + with patch.object(logger._state, "api_client", return_value=mock_api_client): + prompt = braintrust.load_prompt(id="missing-prompt") + with pytest.raises(ValueError, match="Prompt with id missing-prompt not found"): + _ = prompt.id + + def test_load_parameters_uses_explicit_api_key_without_changing_global_login(): simulate_login() original_login_token = logger._state.login_token @@ -518,12 +556,12 @@ def test_load_parameters_uses_explicit_api_key_without_changing_global_login(): def test_load_prompt_does_not_fall_back_to_cache_for_non_transient_errors(server_error): simulate_login() prompt_cache = PromptCache(memory_cache=LRUCache(max_size=10)) - request_conn = MagicMock() - request_conn.get_json.side_effect = [_prompt_response("saved-prompt"), server_error] + request_client = MagicMock() + request_client.openapi.prompts.get_prompt.side_effect = [_prompt_response("saved-prompt"), server_error] with ( patch.object(logger._state, "_prompt_cache", prompt_cache), - patch.object(logger, "_login_loader_conn", return_value=request_conn), + patch.object(logger, "_login_loader_client", return_value=request_client), ): first_prompt = braintrust.load_prompt( project="test-project", @@ -544,12 +582,12 @@ def test_load_prompt_does_not_fall_back_to_cache_for_non_transient_errors(server def test_load_prompt_uses_same_api_keys_cache_for_transient_errors(): simulate_login() prompt_cache = PromptCache(memory_cache=LRUCache(max_size=10)) - request_conn = MagicMock() - request_conn.get_json.side_effect = [_prompt_response("saved-prompt"), _http_error(500)] + request_client = MagicMock() + request_client.openapi.prompts.get_prompt.side_effect = [_prompt_response("saved-prompt"), _http_error(500)] with ( patch.object(logger._state, "_prompt_cache", prompt_cache), - patch.object(logger, "_login_loader_conn", return_value=request_conn), + patch.object(logger, "_login_loader_client", return_value=request_client), ): first_prompt = braintrust.load_prompt( project="test-project", @@ -569,18 +607,18 @@ def test_load_prompt_uses_same_api_keys_cache_for_transient_errors(): def test_load_prompt_does_not_use_another_api_keys_transient_fallback_cache(): simulate_login() prompt_cache = PromptCache(memory_cache=LRUCache(max_size=10)) - first_conn = MagicMock() - first_conn.get_json.return_value = _prompt_response("saved-prompt") - second_conn = MagicMock() - second_conn.get_json.side_effect = _http_error(500) - conns = {"first-api-key": first_conn, "second-api-key": second_conn} + first_client = MagicMock() + first_client.openapi.prompts.get_prompt.return_value = _prompt_response("saved-prompt") + second_client = MagicMock() + second_client.openapi.prompts.get_prompt.side_effect = _http_error(500) + clients = {"first-api-key": first_client, "second-api-key": second_client} def login_for_api_key(options): - return conns[options.api_key] + return clients[options.api_key] with ( patch.object(logger._state, "_prompt_cache", prompt_cache), - patch.object(logger, "_login_loader_conn", side_effect=login_for_api_key), + patch.object(logger, "_login_loader_client", side_effect=login_for_api_key), ): first_prompt = braintrust.load_prompt( project="test-project", @@ -600,22 +638,22 @@ def login_for_api_key(options): @pytest.mark.asyncio async def test_load_prompt_async_eagerly_fetches_prompt(with_simulate_login): - mock_api_conn = MagicMock() - mock_api_conn.get_json.return_value = _prompt_response("saved-prompt") + mock_api_client = MagicMock() + mock_api_client.prompts.get_prompt.return_value = _prompt_response("saved-prompt") - with patch.object(logger._state, "api_conn", return_value=mock_api_conn): + with patch.object(logger._state, "api_client", return_value=mock_api_client): prompt = await braintrust.load_prompt_async( project="test-project", slug="saved-prompt", ) # Unlike load_prompt(), load_prompt_async() resolves the prompt metadata before returning. - mock_api_conn.get_json.assert_called_once_with( - "/v1/prompt", - { - "project_name": "test-project", - "slug": "saved-prompt", - }, + mock_api_client.prompts.get_prompt.assert_called_once_with( + project_name="test-project", + project_id=None, + slug="saved-prompt", + version=None, + environment=None, ) assert prompt.slug == "saved-prompt" assert prompt.build(name="Ada")["messages"][0]["content"] == "Hello Ada" @@ -623,29 +661,29 @@ async def test_load_prompt_async_eagerly_fetches_prompt(with_simulate_login): @pytest.mark.asyncio async def test_load_prompt_async_loads_prompts_in_parallel(with_simulate_login): - mock_api_conn = MagicMock() + mock_api_client = MagicMock() barrier = threading.Barrier(2, timeout=1) - def get_json(_path, args): + def get_prompt(**kwargs): barrier.wait() - return _prompt_response(args["slug"]) + return _prompt_response(kwargs["slug"]) - mock_api_conn.get_json.side_effect = get_json + mock_api_client.prompts.get_prompt.side_effect = get_prompt - with patch.object(logger._state, "api_conn", return_value=mock_api_conn): + with patch.object(logger._state, "api_client", return_value=mock_api_client): prompt1, prompt2 = await asyncio.gather( braintrust.load_prompt_async(project="test-project", slug="prompt-1"), braintrust.load_prompt_async(project="test-project", slug="prompt-2"), ) assert [prompt1.slug, prompt2.slug] == ["prompt-1", "prompt-2"] - assert mock_api_conn.get_json.call_count == 2 + assert mock_api_client.prompts.get_prompt.call_count == 2 class TestLogger(TestCase): def test_load_prompt_prefers_version_over_environment_for_project_slug(self): - mock_api_conn = MagicMock() - mock_api_conn.get_json.return_value = { + mock_api_client = MagicMock() + mock_api_client.prompts.get_prompt.return_value = { "objects": [ { "id": "prompt-123", @@ -667,7 +705,7 @@ def test_load_prompt_prefers_version_over_environment_for_project_slug(self): } simulate_login() - with patch.object(logger._state, "api_conn", return_value=mock_api_conn): + with patch.object(logger._state, "api_client", return_value=mock_api_client): prompt = braintrust.load_prompt( project="test-project", slug="saved-prompt", @@ -676,18 +714,17 @@ def test_load_prompt_prefers_version_over_environment_for_project_slug(self): ) assert prompt.slug == "saved-prompt" - mock_api_conn.get_json.assert_called_once_with( - "/v1/prompt", - { - "project_name": "test-project", - "slug": "saved-prompt", - "version": "v1", - }, + mock_api_client.prompts.get_prompt.assert_called_once_with( + project_name="test-project", + project_id=None, + slug="saved-prompt", + version="v1", + environment=None, ) def test_load_prompt_prefers_version_over_environment_for_id(self): - mock_api_conn = MagicMock() - mock_api_conn.get_json.return_value = { + mock_api_client = MagicMock() + mock_api_client.prompts.get_prompt_id.return_value = { "id": "prompt-123", "project_id": "project-123", "name": "Saved prompt", @@ -705,7 +742,7 @@ def test_load_prompt_prefers_version_over_environment_for_id(self): } simulate_login() - with patch.object(logger._state, "api_conn", return_value=mock_api_conn): + with patch.object(logger._state, "api_client", return_value=mock_api_client): prompt = braintrust.load_prompt( id="prompt-123", version="v1", @@ -713,9 +750,10 @@ def test_load_prompt_prefers_version_over_environment_for_id(self): ) assert prompt.id == "prompt-123" - mock_api_conn.get_json.assert_called_once_with( - "/v1/prompt/prompt-123", - {"version": "v1"}, + mock_api_client.prompts.get_prompt_id.assert_called_once_with( + "prompt-123", + version="v1", + environment=None, ) def test_load_parameters_returns_remote_object(self): diff --git a/py/src/braintrust/type_tests/test_api_client.py b/py/src/braintrust/type_tests/test_api_client.py index 474091d7..3388e014 100644 --- a/py/src/braintrust/type_tests/test_api_client.py +++ b/py/src/braintrust/type_tests/test_api_client.py @@ -7,6 +7,7 @@ CreateDataset, CreateExperiment, CreateProject, + CreatePrompt, Dataset, Experiment, FetchDatasetEventsResponse, @@ -15,11 +16,14 @@ GetDatasetResponse, GetExperimentResponse, GetProjectResponse, + GetPromptResponse, InsertDatasetEventRequest, PatchDataset, PatchExperiment, PatchProject, + PatchPrompt, Project, + Prompt, SummarizeDatasetResponse, SummarizeExperimentResponse, ) @@ -42,6 +46,28 @@ updated_project: Project = openapi_client.projects.patch_project_id(project["id"], body=patch_project) deleted_project: Project = openapi_client.projects.delete_project_id(project["id"]) + create_prompt: CreatePrompt = { + "project_id": project["id"], + "name": "Typed prompt", + "slug": "typed-prompt", + "prompt_data": { + "prompt": { + "type": "chat", + "messages": [{"role": "user", "content": "Hello {{name}}"}], + }, + "options": {"model": "gpt-5-mini"}, + }, + } + prompt: Prompt = openapi_client.prompts.post_prompt(body=create_prompt) + replaced_prompt: Prompt = openapi_client.prompts.put_prompt(body=create_prompt) + prompts: GetPromptResponse = openapi_client.prompts.get_prompt( + project_id=project["id"], slug=prompt["slug"], limit=1 + ) + fetched_prompt: Prompt = openapi_client.prompts.get_prompt_id(prompt["id"], version=prompt["_xact_id"]) + patch_prompt: PatchPrompt = {"description": "updated"} + updated_prompt: Prompt = openapi_client.prompts.patch_prompt_id(prompt["id"], body=patch_prompt) + deleted_prompt: Prompt = openapi_client.prompts.delete_prompt_id(prompt["id"]) + create_dataset: CreateDataset = {"project_id": project["id"], "name": "typed-dataset"} dataset: Dataset = openapi_client.datasets.post_dataset(body=create_dataset) datasets: GetDatasetResponse = openapi_client.datasets.get_dataset(ids=[dataset["id"]], project_id=project["id"]) From 3022c6730445079b1d0fa1dff01aca9ca6d41cee Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 8 Sep 2026 16:48:02 -0400 Subject: [PATCH 2/3] pr review --- py/src/braintrust/api/_transport.py | 19 +++++++++++++++ py/src/braintrust/api/test_client.py | 16 +++++++++---- py/src/braintrust/logger.py | 12 ++++++++-- py/src/braintrust/test_logger.py | 35 +++++++++++++++++++++++++++- 4 files changed, 74 insertions(+), 8 deletions(-) diff --git a/py/src/braintrust/api/_transport.py b/py/src/braintrust/api/_transport.py index 5323d769..a509e91e 100644 --- a/py/src/braintrust/api/_transport.py +++ b/py/src/braintrust/api/_transport.py @@ -222,6 +222,25 @@ def __init__( self._monotonic = monotonic self._wall_clock = wall_clock + def _set_adapter(self, adapter: HTTPAdapter) -> None: + """Install a caller-owned adapter and delegate retries to it.""" + + previous_injected_adapter = self._injected_adapter + replaced_owned_adapters = { + mounted + for mounted in self.session.adapters.values() + if self._owns_session and mounted is not previous_injected_adapter and mounted is not adapter + } + + self._injected_adapter = adapter + self._sdk_retries_enabled = False + self.session.mount("http://", adapter) + self.session.mount("https://", adapter) + + mounted_adapters = set(self.session.adapters.values()) + for replaced_adapter in replaced_owned_adapters - mounted_adapters: + replaced_adapter.close() + def close(self) -> None: if self._owns_session: _unmount_adapter(self.session, self._injected_adapter) diff --git a/py/src/braintrust/api/test_client.py b/py/src/braintrust/api/test_client.py index 7338e6bc..e78c78b6 100644 --- a/py/src/braintrust/api/test_client.py +++ b/py/src/braintrust/api/test_client.py @@ -225,11 +225,12 @@ def test_login_to_state_hydrates_isolated_legacy_connections(monkeypatch): assert isinstance(state.proxy_conn().adapter, RetryRequestExceptionsAdapter) -def test_legacy_adapter_mutation_keeps_characterized_target_scope(monkeypatch): +def test_adapter_mutation_updates_existing_connections_and_generated_client(monkeypatch): state = BraintrustState() state.app_url = "https://app.example.com" state.api_url = "https://api.example.com" state.proxy_url = "https://proxy.example.com" + state._client = BraintrustClient(api_key="secret", api_url=state.api_url) monkeypatch.setattr(logger, "_state", state) monkeypatch.setattr(logger, "_http_adapter", None) @@ -237,11 +238,16 @@ def test_legacy_adapter_mutation_keeps_characterized_target_scope(monkeypatch): api_connection = state.api_conn() proxy_connection = state.proxy_conn() adapter = HTTPAdapter() - logger.set_http_adapter(adapter) + try: + logger.set_http_adapter(adapter) - assert app_connection.adapter is adapter - assert api_connection.adapter is adapter - assert proxy_connection.adapter is None + assert app_connection.adapter is adapter + assert api_connection.adapter is adapter + assert proxy_connection.adapter is None + assert state._client.transport.session.adapters["http://"] is adapter + assert state._client.transport.session.adapters["https://"] is adapter + finally: + state._client.close() def test_state_concurrent_lazy_access_bootstraps_once(monkeypatch): diff --git a/py/src/braintrust/logger.py b/py/src/braintrust/logger.py index 24dd1e20..f5ffdb28 100644 --- a/py/src/braintrust/logger.py +++ b/py/src/braintrust/logger.py @@ -874,6 +874,14 @@ def set_http_adapter(adapter: HTTPAdapter) -> None: if _state._api_conn: _state._api_conn._set_adapter(adapter=adapter) _state._api_conn._reset() + if _state._client: + _state._client.transport._set_adapter(adapter) + + # Per-credential loader resources may have been created with the previous + # adapter. Eviction closes them once any active requests release their lease; + # subsequent loads recreate them with the new global adapter. + _state._loader_login_cache.clear() + _state._loader_api_client_cache.clear() # Sometimes we'd like to launch network requests concurrently. We provide a @@ -2014,8 +2022,8 @@ def _is_loader_cache_fallback_error(error: BaseException) -> bool: if isinstance(current, (json.JSONDecodeError, BraintrustJSONDecodeError)): return False - if isinstance(current, BraintrustTransportError): - return current.retryable + if isinstance(current, BraintrustTransportError) and current.retryable: + return True status_code = getattr(current, "status_code", None) if status_code is None: diff --git a/py/src/braintrust/test_logger.py b/py/src/braintrust/test_logger.py index e41e805a..f44ddaff 100644 --- a/py/src/braintrust/test_logger.py +++ b/py/src/braintrust/test_logger.py @@ -25,6 +25,7 @@ init_logger, logger, ) +from braintrust.api import BraintrustTransportError from braintrust.db_fields import AUDIT_METADATA_FIELD from braintrust.git_fields import GitMetadataSettings, RepoInfo from braintrust.gitutil import get_repo_info @@ -56,7 +57,7 @@ ) from braintrust.util import AugmentedHTTPError from requests import HTTPError -from requests.exceptions import SSLError +from requests.exceptions import ConnectionError, SSLError def test_login_to_state_uses_env_braintrust_api_key(tmp_path, monkeypatch): @@ -579,6 +580,38 @@ def test_load_prompt_does_not_fall_back_to_cache_for_non_transient_errors(server _ = second_prompt.slug +def test_load_prompt_falls_back_to_cache_for_transient_wrapped_transport_errors(): + simulate_login() + prompt_cache = PromptCache(memory_cache=LRUCache(max_size=10)) + request_client = MagicMock() + server_error = BraintrustTransportError( + method="GET", + url="https://api.example.com/v1/prompt", + attempts=1, + retryable=False, + ) + server_error.__cause__ = ConnectionError("custom adapter exhausted its retries") + request_client.openapi.prompts.get_prompt.side_effect = [_prompt_response("saved-prompt"), server_error] + + with ( + patch.object(logger._state, "_prompt_cache", prompt_cache), + patch.object(logger, "_login_loader_client", return_value=request_client), + ): + first_prompt = braintrust.load_prompt( + project="test-project", + slug="saved-prompt", + api_key="prompt-api-key", + ) + assert first_prompt.slug == "saved-prompt" + + cached_prompt = braintrust.load_prompt( + project="test-project", + slug="saved-prompt", + api_key="prompt-api-key", + ) + assert cached_prompt.slug == "saved-prompt" + + def test_load_prompt_uses_same_api_keys_cache_for_transient_errors(): simulate_login() prompt_cache = PromptCache(memory_cache=LRUCache(max_size=10)) From 0cf0199b5fb0f41d1f729c7b0cfc13b2e49ea425 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 8 Sep 2026 19:28:42 -0400 Subject: [PATCH 3/3] better timeout --- py/src/braintrust/api/_transport.py | 16 ++++++++++++++++ py/src/braintrust/api/client.py | 1 + py/src/braintrust/test_logger.py | 19 +++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/py/src/braintrust/api/_transport.py b/py/src/braintrust/api/_transport.py index a509e91e..f178dfb8 100644 --- a/py/src/braintrust/api/_transport.py +++ b/py/src/braintrust/api/_transport.py @@ -1,5 +1,6 @@ """Legacy and policy-aware HTTP transport primitives for the Braintrust SDK.""" +import dataclasses import datetime import http.cookiejar import logging @@ -203,6 +204,7 @@ def __init__( session: requests.Session | None = None, adapter: HTTPAdapter | None = None, enable_sdk_retries: bool | None = None, + request_timeout: float | None = None, persist_cookies: bool = True, sleep: Callable[[float], None] = time.sleep, monotonic: Callable[[], float] = time.monotonic, @@ -215,6 +217,9 @@ def __init__( if not persist_cookies and self._owns_session: self.session.cookies.set_policy(_RejectCookiesPolicy()) self._sdk_retries_enabled = not custom_transport if enable_sdk_retries is None else enable_sdk_retries + if request_timeout is not None and request_timeout <= 0: + raise ValueError("request_timeout must be positive") + self._request_timeout = request_timeout if adapter is not None: self.session.mount("http://", adapter) self.session.mount("https://", adapter) @@ -268,6 +273,17 @@ def request( ) -> requests.Response: method = method.upper() policy = retry_policy or RetryPolicy.for_mode(retry_mode) + if retry_policy is None and self._request_timeout is not None: + max_elapsed_time = policy.max_elapsed_time + if max_elapsed_time is not None: + # Preserve the policy's retry budget beyond the first attempt + # when the configured per-request timeout exceeds its default. + max_elapsed_time += max(0, self._request_timeout - policy.timeout) + policy = dataclasses.replace( + policy, + timeout=self._request_timeout, + max_elapsed_time=max_elapsed_time, + ) replay_safe = retry_mode in (RetryMode.SAFE_READ, RetryMode.IDEMPOTENT_WRITE) body_replayable = _request_body_is_replayable(data, kwargs.get("files")) diff --git a/py/src/braintrust/api/client.py b/py/src/braintrust/api/client.py index 0bce27d4..1206f420 100644 --- a/py/src/braintrust/api/client.py +++ b/py/src/braintrust/api/client.py @@ -37,6 +37,7 @@ def _create_transport( session=session, adapter=adapter, enable_sdk_retries=enable_sdk_retries, + request_timeout=BraintrustEnv.HTTP_TIMEOUT.get(None), persist_cookies=False, ), True, diff --git a/py/src/braintrust/test_logger.py b/py/src/braintrust/test_logger.py index f44ddaff..7f85eef3 100644 --- a/py/src/braintrust/test_logger.py +++ b/py/src/braintrust/test_logger.py @@ -15,6 +15,7 @@ import braintrust import exceptiongroup import pytest +import requests from braintrust import ( Attachment, BaseAttachment, @@ -509,6 +510,24 @@ def test_load_prompt_uses_explicit_api_key_without_changing_global_login(): assert logger._state.login_token == original_login_token +@pytest.mark.parametrize("configured_timeout", [0.25, 120.0]) +def test_load_prompt_preserves_configured_http_timeout(monkeypatch, configured_timeout): + monkeypatch.setenv("BRAINTRUST_HTTP_TIMEOUT", str(configured_timeout)) + simulate_login() + response = requests.Response() + response.status_code = 200 + response.url = "https://api.example.com/v1/prompt" + response.headers["Content-Type"] = "application/json" + response._content = json.dumps(_prompt_response("saved-prompt")).encode() + + assert logger._state._client is not None + with patch.object(logger._state._client.transport.session, "request", return_value=response) as request: + prompt = braintrust.load_prompt(project="test-project", slug="saved-prompt") + assert prompt.slug == "saved-prompt" + + assert request.call_args.kwargs["timeout"] == configured_timeout + + def test_load_prompt_by_id_reports_an_empty_response_as_not_found(): simulate_login() mock_api_client = MagicMock()