Skip to content

chore: validate test session installs #903

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Abhijeet Prasad (AbhiPrasad) merged 2 commits into from
Oct 9, 2026
Merged

chore: validate test session installs #903

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f4bafc4
fix(nox): validate test session installs
AbhiPrasad Oct 9, 2026
4b735a7
fix(nox): resolve session dependency conflicts
AbhiPrasad Oct 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 43 additions & 29 deletions py/noxfile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,19 +153,23 @@ def _get_matrix_versions(prefix: str) -> tuple[str, ...]:

Returns a tuple ordered with LATEST first, then descending version order.
"""
matrix_entry = _MATRIX.get(prefix, {})
matrix_entry = _MATRIX.get(prefix)
if not matrix_entry:
raise KeyError(f"Missing [tool.braintrust.matrix.{prefix}] in pyproject.toml")
latest = [LATEST] if "latest" in matrix_entry else []
rest = sorted([v for v in matrix_entry if v != "latest"], key=Version, reverse=True)
return tuple(latest + rest)


def _install_matrix_dep(session: nox.Session, prefix: str, version: str, constraint_group: str | None = None) -> None:
"""Install a matrix dependency, optionally using pinned compatibility constraints."""
matrix_entry = _MATRIX.get(prefix, {})
matrix_entry = _MATRIX.get(prefix)
if not matrix_entry:
session.error(f"Missing [tool.braintrust.matrix.{prefix}] in pyproject.toml")
key = "latest" if version == LATEST else version
req = matrix_entry.get(key)
if not req:
return
session.error(f"Missing matrix key {key!r} in [tool.braintrust.matrix.{prefix}]")
if constraint_group is None:
session.install(req, silent=SILENT_INSTALLS)
return
Expand Down Expand Up @@ -298,11 +302,10 @@ def test_bedrock_runtime(session, version):
@nox.parametrize("version", INSTRUCTOR_VERSIONS, ids=INSTRUCTOR_VERSIONS)
def test_instructor(session, version):
"""Test the native Instructor structured-output integration."""
_install_test_deps(session)
_install_test_deps(session, "test-instructor")
_install_matrix_dep(session, "instructor", version)
# Instructor wraps a provider client; we exercise it against an OpenAI
# client via VCR cassettes recorded against ``api.openai.com``.
_install_matrix_dep(session, "openai", LATEST)
# The locked auxiliary group selects an OpenAI SDK version supported by
# Instructor. These tests use VCR cassettes against ``api.openai.com``.
_run_tests(session, f"{INTEGRATION_DIR}/instructor/test_instructor.py", version=version)


Expand Down Expand Up @@ -341,9 +344,8 @@ def test_openai_http2_streaming(session, version):
@nox.parametrize("version", OPENAI_ENDPOINT_VERSIONS, ids=OPENAI_ENDPOINT_VERSIONS)
def test_btx_openai(session, version):
"""Run the BTX cross-language LLM-span spec tests (OpenAI provider)."""
_install_test_deps(session)
_install_test_deps(session, "test-btx")
_install_matrix_dep(session, "openai", version)
session.install("pyyaml")
_run_tests(session, "braintrust/btx", version=version, env={"BTX_PROVIDER": "openai", "BTX_CLIENT": "openai"})


Expand Down Expand Up @@ -447,6 +449,17 @@ def test_livekit_agents(session, version):
_install_test_deps(session)
_install_matrix_dep(session, "livekit-agents", version)
_install_group_locked(session, "test-livekit-agents")
if version == "1.3.1":
# LiveKit 1.3.1 accepts a newer OTLP exporter, but its dependencies
# require an SDK version that breaks LiveKit's LogData import. The
# locked group downgrades the exporter, so remove its now-orphaned deps.
session.run(
"uv",
"pip",
"uninstall",
"opentelemetry-exporter-otlp-common",
"opentelemetry-exporter-http-transport",
)
livekit_server_dir = _ensure_livekit_server(session)
env = {
"LIVEKIT_URL": os.environ.get("LIVEKIT_URL", "ws://localhost:7880"),
Expand Down Expand Up @@ -541,9 +554,8 @@ def test_pydantic_ai_integration(session, version):
@nox.parametrize("version", PYDANTIC_AI_INTEGRATION_VERSIONS, ids=PYDANTIC_AI_INTEGRATION_VERSIONS)
def test_pydantic_ai_logfire(session, version):
"""Test pydantic_ai + logfire coexistence (issue #1324)."""
_install_test_deps(session)
_install_matrix_dep(session, "pydantic-ai-integration", version)
_install_group_locked(session, "test-pydantic-ai-logfire")
_install_test_deps(session, "test-pydantic-ai-logfire")
_install_matrix_dep(session, "pydantic-ai-integration", version, "test-pydantic-ai-logfire-constraints")
_run_tests(session, f"{INTEGRATION_DIR}/pydantic_ai/test_pydantic_ai_logfire.py", version=version)


Expand Down Expand Up @@ -682,12 +694,9 @@ def test_deepagents(session, version):
@nox.session()
@nox.parametrize("version", LLAMAINDEX_VERSIONS, ids=LLAMAINDEX_VERSIONS)
def test_llamaindex(session, version):
_install_test_deps(session)
_install_group_locked(session, "test-llamaindex")
group = "test-llamaindex-0-13" if version == "0.13.0" else "test-llamaindex"
_install_test_deps(session, group)
_install_matrix_dep(session, "llama-index-core", version, "test-sqlalchemy-2-0")
# These packages are tightly version-coupled to llama-index-core, so we
# install them unpinned and let pip resolve compatible versions.
session.install("llama-index-llms-openai", "llama-index-embeddings-openai", silent=SILENT_INSTALLS)
_run_tests(session, f"{INTEGRATION_DIR}/llamaindex/test_llamaindex.py", version=version)


Expand Down Expand Up @@ -823,6 +832,7 @@ def test_api_codegen(session):
"""Test the pinned OpenAPI validator and deterministic model generator."""
_install_test_deps(session)
_install_group_locked(session, "api-codegen")
_check_installed_deps(session)
session.run("pytest", "-p", "no:braintrust", "tests/api_codegen", *session.posargs)


Expand All @@ -838,9 +848,7 @@ def test_braintrust_core(session):
@nox.session()
def test_cli(session):
"""Test CLI/devserver with starlette installed."""
_install_test_deps(session)
session.install(".[cli]")
_install_group_locked(session, "test-cli")
_install_test_deps(session, "test-cli")
_run_tests(session, DEVSERVER_DIR)


Expand Down Expand Up @@ -874,6 +882,7 @@ def test_types(session):
"""Run type-check tests with pyright, mypy, and pytest."""
_install_test_deps(session)
_install_group_locked(session, "test-types")
_check_installed_deps(session)

type_tests_dir = f"src/{TYPE_TESTS_DIR}"
test_files = glob.glob(os.path.join(type_tests_dir, "test_*.py"))
Expand All @@ -895,26 +904,25 @@ def test_types(session):

@nox.session()
def pylint(session):
# Install the project without optional extras, then install stable lint and
# test dependencies from the lockfile. Optional vendor packages follow the
# matrix's latest pins so lint sees the same SDK versions as CI tests.
# Install the project and locked base/lint dependencies, then vendor SDKs
# at their matrix latest versions for import coverage.
session.install(".")
_install_group_locked(session, "test", "lint")
# Provider latest pins intentionally coexist in the lint environment even
# when their dependency metadata conflicts. Validate the lock-resolved base
# environment before adding those matrix versions for import coverage.
_check_installed_deps(session)
minimum_python = {
"ai-sdk": (3, 12),
"agentscope": (3, 11),
"harbor": (3, 12),
"pipecat-ai": (3, 11),
}
for package in _VENDOR_TABLE:
# These providers do not support the Python versions in the static
# checks matrix. Keep lint installs compatible with the active Python.
if sys.version_info[:2] < minimum_python.get(package, (3, 10)):
continue
if package == "pipecat-ai" and sys.version_info >= (3, 14):
continue
# CrewAI's latest release brings unsuitable chromadb transitive deps;
# its integration tests install it in their isolated session instead.
if package == "crewai":
continue
_install_matrix_dep(session, package, LATEST)
Expand All @@ -939,7 +947,7 @@ def pylint(session):
session.run("pylint", "--errors-only", *files)


def _install_test_deps(session):
def _install_test_deps(session, *groups):
# Choose the way we'll install braintrust ... wheel or source.
install_wheel = "--wheel" in session.posargs

Expand All @@ -954,7 +962,7 @@ def _install_test_deps(session):

# Install base test deps (pytest, pytest-asyncio, pytest-vcr) from the
# lockfile so transitive deps are pinned and reproducible.
_install_group_locked(session, "test")
_install_group_locked(session, "test", *groups)

# Sanity check braintrust imports from where this mode expects it:
# site-packages for a wheel, the source tree for an editable install.
Expand All @@ -966,6 +974,11 @@ def _install_test_deps(session):
session.run("python", "-c", ";".join(lines))


def _check_installed_deps(session):
"""Fail setup if installed packages have incompatible requirements."""
session.run("uv", "pip", "check")


def _get_braintrust_wheel():
path = "dist/braintrust-*.whl"
wheels = glob.glob(path)
Expand Down Expand Up @@ -1015,6 +1028,7 @@ def _run_tests(
run_from_temp_dir=False,
):
"""Run tests against a wheel or the source code. Paths should be relative and start with braintrust."""
_check_installed_deps(session)
env = env.copy() if env else {}
if version:
env["BRAINTRUST_TEST_PACKAGE_VERSION"] = version
Expand Down
86 changes: 76 additions & 10 deletions py/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,11 @@ test-strands = [

test-pydantic-ai-logfire = [
{include-group = "test"},
"logfire==4.32.1",
"logfire==4.39.0",
]

test-pydantic-ai-logfire-constraints = [
"logfire==4.39.0",
]

test-langchain = [
Expand All @@ -193,6 +197,8 @@ test-livekit-agents = [
# livekit-agents 1.3.x imports opentelemetry.sdk._logs.LogData, removed in newer SDKs.
# This is a LiveKit runtime compatibility constraint, not a Braintrust OTel integration dependency.
"opentelemetry-sdk<1.39",
"opentelemetry-api<1.39",
"opentelemetry-exporter-otlp<1.39",
]

test-pipecat = [
Expand All @@ -212,6 +218,26 @@ test-crewai = [

test-llamaindex = [
{include-group = "test"},
"llama-index-llms-openai",
"llama-index-embeddings-openai",
Comment thread
AbhiPrasad marked this conversation as resolved.
]

test-llamaindex-0-13 = [
{include-group = "test"},
"llama-index-core==0.13.0",
"llama-index-llms-openai==0.5.0",
"llama-index-embeddings-openai==0.5.0",
]

test-btx = [
{include-group = "test"},
"pyyaml",
]

test-instructor = [
{include-group = "test"},
"jiter<0.11",
"openai==1.99.9",
]

test-huggingface-hub = [
Expand All @@ -236,7 +262,12 @@ test-transformers = [

test-cli = [
{include-group = "test"},
"boto3",
"httpx==0.28.1",
"python-dotenv",
"starlette",
"uv",
"uvicorn",
]

test-types = [
Expand All @@ -251,15 +282,32 @@ test-types = [
lint = [
"pylint==4.0.5",
"pyperf==2.10.0",
# Non-vendor packages needed by pylint imports and the project's `all` extra.
# Vendor SDK versions are installed from [tool.braintrust.matrix] below.
"python-dotenv",
"starlette",
"uv",
"uvicorn",
"orjson; platform_python_implementation != 'PyPy'",
# Imported directly by LiveKit integration tests, but not a standalone
# version-matrix package.
# Packages below are needed so pylint can resolve every SDK import.
# Versions are unpinned (unless already constrained) so the resolver
# can find mutually compatible transitive deps.
"agentscope<2",
"agno",
"ai>=0.4.0; python_version>='3.12'",
"anthropic",
"autogen-agentchat",
"autogen-ext[openai]",
"claude-agent-sdk",
"cohere",
"autoevals",
"braintrust-core",
# Do not install crewai in the shared lint environment: current releases
# transitively pin vulnerable chromadb versions. CrewAI integration tests
# install the provider package via the nox matrix instead.
# onnxruntime 1.24.3 dropped cp310 wheels. Constrain the transitive
# resolution on Python 3.10 so pylint can still build its environment.
"onnxruntime<1.24; python_version<'3.11'",
"dspy",
"google-adk",
"google-genai",
"google-cloud-discoveryengine",
"instructor",
"litellm>=1.104.2",
"livekit-agents>=1.8.5",
"livekit-plugins-openai",
# Keep these available to pylint without installing CrewAI (whose
# transitive chromadb versions are unsuitable for shared CI).
Expand All @@ -271,6 +319,13 @@ lint = [
"llama-index-core",
"llama-index-llms-openai",
"llama-index-embeddings-openai",
# Keep the lint environment's optional SDK dependencies lock-resolved
# instead of installing the unconstrained [all] extra separately.
"opentelemetry-api==1.39.1",
"opentelemetry-exporter-otlp-proto-http==1.39.1",
"opentelemetry-sdk==1.39.1",
"orjson",
{include-group = "test-cli"},
]

# -- Build and code-generation deps --------------------------------------------
Expand Down Expand Up @@ -326,6 +381,9 @@ conflicts = [
{group = "test-strands"},
{group = "test-langchain"},
{group = "test-deepagents"},
{group = "test-llamaindex"},
{group = "test-llamaindex-0-13"},
{group = "test-instructor"},
{group = "test-livekit-agents"},
{group = "lint"},
],
Expand All @@ -339,6 +397,14 @@ conflicts = [
{group = "test-pydantic-ai-otel-events"},
{group = "test-livekit-agents"},
],
[
{group = "test-pydantic-ai-logfire-constraints"},
{group = "test-livekit-agents"},
],
[
{group = "test-llamaindex"},
{group = "test-llamaindex-0-13"},
],
]

# ---------------------------------------------------------------------------
Expand Down
Loading
Loading