From 006e70691a7cc857cbfd9c7818338d12a2bf0f84 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Mon, 1 Jun 2026 10:45:18 -0300 Subject: [PATCH] fix(runtime): auto-derive AZURE_OPENAI_ENDPOINT from project endpoint CONTRIBUTING.md and the env-var docs both state that AZURE_OPENAI_ENDPOINT is auto-derived from the Foundry project endpoint when absent, but pipeline/runtime.py::_model_config only read AZURE_OPENAI_ENDPOINT with no fallback. A fresh workspace created by `agentops init` (which writes AZURE_AI_FOUNDRY_PROJECT_ENDPOINT but not AZURE_OPENAI_ENDPOINT) therefore always tripped 'Missing environment variables: AZURE_OPENAI_ENDPOINT' the first time AI-assisted evaluators tried to run locally. Adds derive_openai_endpoint_from_project() in utils/azure_endpoints.py that trims the trailing /api/projects/ segment to recover the AI Services account base URL. _model_config uses it as a fallback, restoring the documented behavior. The missing-deployment error also now nudges users toward the Foundry deployments list and the execution: cloud escape hatch so the next step is obvious without leaving the terminal. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 24 ++++++ src/agentops/pipeline/runtime.py | 24 +++++- src/agentops/utils/azure_endpoints.py | 32 ++++++++ tests/unit/test_azure_endpoints.py | 65 ++++++++++++++- tests/unit/test_runtime_model_config.py | 100 ++++++++++++++++++++++++ 5 files changed, 242 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_runtime_model_config.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8deaa3a..b2ea180 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,30 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] +### Fixed +- **`agentops eval run` in local execution mode no longer fails with + `Missing environment variables: AZURE_OPENAI_ENDPOINT` when only the + Foundry project endpoint is configured.** `CONTRIBUTING.md` and the + user-facing env-var docs both stated that `AZURE_OPENAI_ENDPOINT` is + "auto-derived from the project endpoint when absent", but + `pipeline/runtime.py::_model_config` only read the explicit + `AZURE_OPENAI_ENDPOINT` env var with no fallback — so a fresh workspace + created by `agentops init` (which writes `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` + but not `AZURE_OPENAI_ENDPOINT`) would always trip the missing-env error + the first time AI-assisted evaluators tried to run locally. The new + helper `agentops.utils.azure_endpoints.derive_openai_endpoint_from_project` + trims the trailing `/api/projects/` segment from a Foundry project + URL (covering both `services.ai.azure.com` and the legacy + `cognitiveservices.azure.com` hosts) to recover the AI Services account + base URL, which is exactly what the `openai` and `azure-ai-evaluation` + SDKs want. `_model_config` now uses the derived value as a fallback + whenever `AZURE_OPENAI_ENDPOINT` is unset, so the documented behavior + finally matches the runtime. When `AZURE_OPENAI_DEPLOYMENT` is the only + thing missing, the error message now points users at the deployment list + in the Foundry portal *and* mentions the `execution: cloud` escape hatch + in `agentops.yaml` so the next step is obvious without leaving the + terminal. + ## [0.3.3] - 2026-05-31 ### Changed diff --git a/src/agentops/pipeline/runtime.py b/src/agentops/pipeline/runtime.py index 23da6af..c3cb675 100644 --- a/src/agentops/pipeline/runtime.py +++ b/src/agentops/pipeline/runtime.py @@ -65,10 +65,22 @@ def _credential() -> Any: def _model_config() -> Dict[str, str]: - from agentops.utils.azure_endpoints import normalize_azure_openai_endpoint + from agentops.utils.azure_endpoints import ( + derive_openai_endpoint_from_project, + normalize_azure_openai_endpoint, + ) raw_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") endpoint = normalize_azure_openai_endpoint(raw_endpoint) + if not endpoint: + # CONTRIBUTING.md promises ``AZURE_OPENAI_ENDPOINT`` is "auto-derived + # from the project endpoint when absent". The Foundry project URL + # already encodes the AI Services account host, so we can recover + # the base inference endpoint without an extra round-trip or any + # new wizard prompt. + endpoint = derive_openai_endpoint_from_project( + os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") + ) deployment = os.getenv("AZURE_OPENAI_DEPLOYMENT") or os.getenv( "AZURE_AI_MODEL_DEPLOYMENT_NAME" ) @@ -85,9 +97,17 @@ def _model_config() -> Dict[str, str]: if not deployment: missing.append("AZURE_OPENAI_DEPLOYMENT") if missing: + hint = "" + if "AZURE_OPENAI_DEPLOYMENT" in missing: + hint = ( + " Set AZURE_OPENAI_DEPLOYMENT to the name of a model " + "deployment in your Foundry project (Models + endpoints " + "in the portal), or switch the run to `execution: cloud` " + "in agentops.yaml so Foundry runs the evaluators server-side." + ) raise RuntimeError( "AI-assisted evaluators require an evaluator model. " - "Missing environment variables: " + ", ".join(missing) + "Missing environment variables: " + ", ".join(missing) + "." + hint ) config: Dict[str, str] = { diff --git a/src/agentops/utils/azure_endpoints.py b/src/agentops/utils/azure_endpoints.py index 44f504c..fb36179 100644 --- a/src/agentops/utils/azure_endpoints.py +++ b/src/agentops/utils/azure_endpoints.py @@ -60,3 +60,35 @@ def normalize_azure_openai_endpoint(value: Optional[str]) -> Optional[str]: # Trim any straggling trailing slash so callers building paths # never get a doubled ``//``. return rewritten.rstrip("/") + + +# Foundry project endpoints look like +# ``https://.services.ai.azure.com/api/projects/`` (or the +# legacy ``cognitiveservices.azure.com`` host). The Azure OpenAI inference +# endpoint that the ``openai`` and ``azure-ai-evaluation`` SDKs want is the +# *account* base URL — i.e. the same scheme/host with the project path +# trimmed off. ``derive_openai_endpoint_from_project`` performs that trim so +# users who only configure ``AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`` (the value +# the ``agentops init`` wizard already writes) get a working AI-assisted +# evaluator run without having to hand-author ``AZURE_OPENAI_ENDPOINT`` too. +_PROJECT_PATH_RE = re.compile(r"/api/projects/[^/?#]+/*\Z", re.IGNORECASE) + + +def derive_openai_endpoint_from_project(value: Optional[str]) -> Optional[str]: + """Return the Azure OpenAI base URL embedded in a Foundry project endpoint. + + The function is conservative: it only rewrites URLs whose final path + segment matches ``/api/projects/`` exactly. Anything else + (already-base URLs, URLs with extra path segments after the project + name, non-Foundry hosts) is returned untouched apart from a normalizing + pass through :func:`normalize_azure_openai_endpoint`. ``None`` and + empty strings pass through unchanged so callers can keep the + ``os.getenv`` ergonomic of ``derive_openai_endpoint_from_project(os.getenv(...))``. + """ + if value is None: + return None + stripped = value.strip() + if not stripped: + return stripped + trimmed = _PROJECT_PATH_RE.sub("", stripped) + return normalize_azure_openai_endpoint(trimmed) diff --git a/tests/unit/test_azure_endpoints.py b/tests/unit/test_azure_endpoints.py index 552df13..08f7d29 100644 --- a/tests/unit/test_azure_endpoints.py +++ b/tests/unit/test_azure_endpoints.py @@ -4,7 +4,10 @@ import pytest -from agentops.utils.azure_endpoints import normalize_azure_openai_endpoint +from agentops.utils.azure_endpoints import ( + derive_openai_endpoint_from_project, + normalize_azure_openai_endpoint, +) @pytest.mark.parametrize( @@ -52,3 +55,63 @@ def test_normalize_only_strips_at_the_end() -> None: # A path containing "openai" earlier in the URL must not be touched. raw = "https://proxy.example.com/openai/v1/extra" assert normalize_azure_openai_endpoint(raw) == raw.rstrip("/") + + +@pytest.mark.parametrize( + "raw, expected", + [ + # Canonical Foundry project endpoint (services.ai.azure.com host). + ( + "https://ai-account-xyz.services.ai.azure.com/api/projects/proj-default", + "https://ai-account-xyz.services.ai.azure.com", + ), + # Trailing slash on the project segment. + ( + "https://ai-account-xyz.services.ai.azure.com/api/projects/proj-default/", + "https://ai-account-xyz.services.ai.azure.com", + ), + # Legacy cognitiveservices host. + ( + "https://acct.cognitiveservices.azure.com/api/projects/p1", + "https://acct.cognitiveservices.azure.com", + ), + # Project names may include hyphens and digits. + ( + "https://acct.services.ai.azure.com/api/projects/travel-agent-sandbox", + "https://acct.services.ai.azure.com", + ), + # Case-insensitive match on the ``/api/projects/`` segment. + ( + "https://acct.services.ai.azure.com/API/Projects/p1", + "https://acct.services.ai.azure.com", + ), + # Already a base URL — passed through unchanged (apart from trailing slash). + ( + "https://acct.services.ai.azure.com", + "https://acct.services.ai.azure.com", + ), + # Already a base URL with the OpenAI inference suffix — still normalized. + ( + "https://acct.services.ai.azure.com/openai/v1", + "https://acct.services.ai.azure.com", + ), + ], +) +def test_derive_openai_endpoint_from_project(raw: str, expected: str) -> None: + assert derive_openai_endpoint_from_project(raw) == expected + + +def test_derive_openai_endpoint_from_project_passes_through_none() -> None: + assert derive_openai_endpoint_from_project(None) is None + + +def test_derive_openai_endpoint_from_project_passes_through_empty() -> None: + assert derive_openai_endpoint_from_project("") == "" + assert derive_openai_endpoint_from_project(" ") == "" + + +def test_derive_openai_endpoint_from_project_keeps_extra_path() -> None: + # Only the final ``/api/projects/`` segment is trimmed; an endpoint + # carrying additional sub-paths (e.g. an explicit agent path) is left as-is. + raw = "https://acct.services.ai.azure.com/api/projects/proj/agents/foo" + assert derive_openai_endpoint_from_project(raw) == raw.rstrip("/") diff --git a/tests/unit/test_runtime_model_config.py b/tests/unit/test_runtime_model_config.py new file mode 100644 index 0000000..dfc1502 --- /dev/null +++ b/tests/unit/test_runtime_model_config.py @@ -0,0 +1,100 @@ +"""Tests for :func:`agentops.pipeline.runtime._model_config`. + +The covered behavior: + +* ``AZURE_OPENAI_ENDPOINT`` is preferred when explicitly set. +* When absent, the endpoint is auto-derived from + ``AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`` — the CONTRIBUTING-stated promise + that ``agentops init`` already satisfies for the project endpoint. +* When neither is available the error mentions both missing variables and + hints the ``execution: cloud`` escape hatch for the deployment-only case. +""" + +from __future__ import annotations + +import pytest + +from agentops.pipeline.runtime import _model_config + + +@pytest.fixture(autouse=True) +def _clear_env(monkeypatch: pytest.MonkeyPatch) -> None: + for key in ( + "AZURE_OPENAI_ENDPOINT", + "AZURE_OPENAI_DEPLOYMENT", + "AZURE_AI_MODEL_DEPLOYMENT_NAME", + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", + "AZURE_OPENAI_API_VERSION", + ): + monkeypatch.delenv(key, raising=False) + + +def test_explicit_openai_endpoint_wins(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AZURE_OPENAI_ENDPOINT", "https://explicit.openai.azure.com") + monkeypatch.setenv("AZURE_OPENAI_DEPLOYMENT", "gpt-4o-mini") + monkeypatch.setenv( + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", + "https://different.services.ai.azure.com/api/projects/p1", + ) + + cfg = _model_config() + + assert cfg["azure_endpoint"] == "https://explicit.openai.azure.com" + assert cfg["azure_deployment"] == "gpt-4o-mini" + + +def test_endpoint_auto_derived_from_project(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv( + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", + "https://ai-account-xyz.services.ai.azure.com/api/projects/proj-default", + ) + monkeypatch.setenv("AZURE_OPENAI_DEPLOYMENT", "gpt-4o-mini") + + cfg = _model_config() + + assert cfg["azure_endpoint"] == "https://ai-account-xyz.services.ai.azure.com" + assert cfg["azure_deployment"] == "gpt-4o-mini" + + +def test_missing_deployment_still_raises_with_hint(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv( + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", + "https://acct.services.ai.azure.com/api/projects/p1", + ) + + with pytest.raises(RuntimeError) as excinfo: + _model_config() + + message = str(excinfo.value) + assert "AZURE_OPENAI_DEPLOYMENT" in message + assert "AZURE_OPENAI_ENDPOINT" not in message + # The hint nudges the user toward the cloud-execution escape hatch. + assert "execution: cloud" in message + + +def test_missing_both_lists_endpoint_and_deployment(monkeypatch: pytest.MonkeyPatch) -> None: + with pytest.raises(RuntimeError) as excinfo: + _model_config() + + message = str(excinfo.value) + assert "AZURE_OPENAI_ENDPOINT" in message + assert "AZURE_OPENAI_DEPLOYMENT" in message + + +def test_default_api_version_pinned(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AZURE_OPENAI_ENDPOINT", "https://x.openai.azure.com") + monkeypatch.setenv("AZURE_OPENAI_DEPLOYMENT", "gpt-4o-mini") + + cfg = _model_config() + + assert cfg["api_version"] == "2025-04-01-preview" + + +def test_api_version_override(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AZURE_OPENAI_ENDPOINT", "https://x.openai.azure.com") + monkeypatch.setenv("AZURE_OPENAI_DEPLOYMENT", "gpt-4o-mini") + monkeypatch.setenv("AZURE_OPENAI_API_VERSION", "2024-12-01-preview") + + cfg = _model_config() + + assert cfg["api_version"] == "2024-12-01-preview"