fix(runtime): auto-derive AZURE_OPENAI_ENDPOINT from project endpoint

CHANGELOG.md

@@ -5,6 +5,30 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres
 
 ## [Unreleased]
 
+### Fixed
+- **`agentops eval run` in local execution mode no longer fails with
+  `Missing environment variables: AZURE_OPENAI_ENDPOINT` when only the
+  Foundry project endpoint is configured.** `CONTRIBUTING.md` and the
+  user-facing env-var docs both stated that `AZURE_OPENAI_ENDPOINT` is
+  "auto-derived from the project endpoint when absent", but
+  `pipeline/runtime.py::_model_config` only read the explicit
+  `AZURE_OPENAI_ENDPOINT` env var with no fallback — so a fresh workspace
+  created by `agentops init` (which writes `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`
+  but not `AZURE_OPENAI_ENDPOINT`) would always trip the missing-env error
+  the first time AI-assisted evaluators tried to run locally. The new
+  helper `agentops.utils.azure_endpoints.derive_openai_endpoint_from_project`
+  trims the trailing `/api/projects/<name>` segment from a Foundry project
+  URL (covering both `services.ai.azure.com` and the legacy
+  `cognitiveservices.azure.com` hosts) to recover the AI Services account
+  base URL, which is exactly what the `openai` and `azure-ai-evaluation`
+  SDKs want. `_model_config` now uses the derived value as a fallback
+  whenever `AZURE_OPENAI_ENDPOINT` is unset, so the documented behavior
+  finally matches the runtime. When `AZURE_OPENAI_DEPLOYMENT` is the only
+  thing missing, the error message now points users at the deployment list
+  in the Foundry portal *and* mentions the `execution: cloud` escape hatch
+  in `agentops.yaml` so the next step is obvious without leaving the
+  terminal.
+
 ## [0.3.3] - 2026-05-31
 
 ### Changed

src/agentops/pipeline/runtime.py

@@ -65,10 +65,22 @@ def _credential() -> Any:
 
 
 def _model_config() -> Dict[str, str]:
-    from agentops.utils.azure_endpoints import normalize_azure_openai_endpoint
+    from agentops.utils.azure_endpoints import (
+        derive_openai_endpoint_from_project,
+        normalize_azure_openai_endpoint,
+    )
 
     raw_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")
     endpoint = normalize_azure_openai_endpoint(raw_endpoint)
+    if not endpoint:
+        # CONTRIBUTING.md promises ``AZURE_OPENAI_ENDPOINT`` is "auto-derived
+        # from the project endpoint when absent". The Foundry project URL
+        # already encodes the AI Services account host, so we can recover
+        # the base inference endpoint without an extra round-trip or any
+        # new wizard prompt.
+        endpoint = derive_openai_endpoint_from_project(
+            os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT")
+        )
     deployment = os.getenv("AZURE_OPENAI_DEPLOYMENT") or os.getenv(
         "AZURE_AI_MODEL_DEPLOYMENT_NAME"
     )
@@ -85,9 +97,17 @@ def _model_config() -> Dict[str, str]:
     if not deployment:
         missing.append("AZURE_OPENAI_DEPLOYMENT")
     if missing:
+        hint = ""
+        if "AZURE_OPENAI_DEPLOYMENT" in missing:
+            hint = (
+                " Set AZURE_OPENAI_DEPLOYMENT to the name of a model "
+                "deployment in your Foundry project (Models + endpoints "
+                "in the portal), or switch the run to `execution: cloud` "
+                "in agentops.yaml so Foundry runs the evaluators server-side."
+            )
         raise RuntimeError(
             "AI-assisted evaluators require an evaluator model. "
-            "Missing environment variables: " + ", ".join(missing)
+            "Missing environment variables: " + ", ".join(missing) + "." + hint
         )
 
     config: Dict[str, str] = {

src/agentops/utils/azure_endpoints.py

@@ -60,3 +60,35 @@ def normalize_azure_openai_endpoint(value: Optional[str]) -> Optional[str]:
     # Trim any straggling trailing slash so callers building paths
     # never get a doubled ``//``.
     return rewritten.rstrip("/")
+
+
+# Foundry project endpoints look like
+# ``https://<account>.services.ai.azure.com/api/projects/<project>`` (or the
+# legacy ``cognitiveservices.azure.com`` host). The Azure OpenAI inference
+# endpoint that the ``openai`` and ``azure-ai-evaluation`` SDKs want is the
+# *account* base URL — i.e. the same scheme/host with the project path
+# trimmed off. ``derive_openai_endpoint_from_project`` performs that trim so
+# users who only configure ``AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`` (the value
+# the ``agentops init`` wizard already writes) get a working AI-assisted
+# evaluator run without having to hand-author ``AZURE_OPENAI_ENDPOINT`` too.
+_PROJECT_PATH_RE = re.compile(r"/api/projects/[^/?#]+/*\Z", re.IGNORECASE)
+
+
+def derive_openai_endpoint_from_project(value: Optional[str]) -> Optional[str]:
+    """Return the Azure OpenAI base URL embedded in a Foundry project endpoint.
+
+    The function is conservative: it only rewrites URLs whose final path
+    segment matches ``/api/projects/<project>`` exactly. Anything else
+    (already-base URLs, URLs with extra path segments after the project
+    name, non-Foundry hosts) is returned untouched apart from a normalizing
+    pass through :func:`normalize_azure_openai_endpoint`. ``None`` and
+    empty strings pass through unchanged so callers can keep the
+    ``os.getenv`` ergonomic of ``derive_openai_endpoint_from_project(os.getenv(...))``.
+    """
+    if value is None:
+        return None
+    stripped = value.strip()
+    if not stripped:
+        return stripped
+    trimmed = _PROJECT_PATH_RE.sub("", stripped)
+    return normalize_azure_openai_endpoint(trimmed)

tests/unit/test_azure_endpoints.py

@@ -4,7 +4,10 @@
 
 import pytest
 
-from agentops.utils.azure_endpoints import normalize_azure_openai_endpoint
+from agentops.utils.azure_endpoints import (
+    derive_openai_endpoint_from_project,
+    normalize_azure_openai_endpoint,
+)
 
 
 @pytest.mark.parametrize(
@@ -52,3 +55,63 @@ def test_normalize_only_strips_at_the_end() -> None:
     # A path containing "openai" earlier in the URL must not be touched.
     raw = "https://proxy.example.com/openai/v1/extra"
     assert normalize_azure_openai_endpoint(raw) == raw.rstrip("/")
+
+
+@pytest.mark.parametrize(
+    "raw, expected",
+    [
+        # Canonical Foundry project endpoint (services.ai.azure.com host).
+        (
+            "https://ai-account-xyz.services.ai.azure.com/api/projects/proj-default",
+            "https://ai-account-xyz.services.ai.azure.com",
+        ),
+        # Trailing slash on the project segment.
+        (
+            "https://ai-account-xyz.services.ai.azure.com/api/projects/proj-default/",
+            "https://ai-account-xyz.services.ai.azure.com",
+        ),
+        # Legacy cognitiveservices host.
+        (
+            "https://acct.cognitiveservices.azure.com/api/projects/p1",
+            "https://acct.cognitiveservices.azure.com",
+        ),
+        # Project names may include hyphens and digits.
+        (
+            "https://acct.services.ai.azure.com/api/projects/travel-agent-sandbox",
+            "https://acct.services.ai.azure.com",
+        ),
+        # Case-insensitive match on the ``/api/projects/`` segment.
+        (
+            "https://acct.services.ai.azure.com/API/Projects/p1",
+            "https://acct.services.ai.azure.com",
+        ),
+        # Already a base URL — passed through unchanged (apart from trailing slash).
+        (
+            "https://acct.services.ai.azure.com",
+            "https://acct.services.ai.azure.com",
+        ),
+        # Already a base URL with the OpenAI inference suffix — still normalized.
+        (
+            "https://acct.services.ai.azure.com/openai/v1",
+            "https://acct.services.ai.azure.com",
+        ),
+    ],
+)
+def test_derive_openai_endpoint_from_project(raw: str, expected: str) -> None:
+    assert derive_openai_endpoint_from_project(raw) == expected
+
+
+def test_derive_openai_endpoint_from_project_passes_through_none() -> None:
+    assert derive_openai_endpoint_from_project(None) is None
+
+
+def test_derive_openai_endpoint_from_project_passes_through_empty() -> None:
+    assert derive_openai_endpoint_from_project("") == ""
+    assert derive_openai_endpoint_from_project("   ") == ""
+
+
+def test_derive_openai_endpoint_from_project_keeps_extra_path() -> None:
+    # Only the final ``/api/projects/<name>`` segment is trimmed; an endpoint
+    # carrying additional sub-paths (e.g. an explicit agent path) is left as-is.
+    raw = "https://acct.services.ai.azure.com/api/projects/proj/agents/foo"
+    assert derive_openai_endpoint_from_project(raw) == raw.rstrip("/")

tests/unit/test_runtime_model_config.py

@@ -0,0 +1,100 @@
+"""Tests for :func:`agentops.pipeline.runtime._model_config`.
+
+The covered behavior:
+
+* ``AZURE_OPENAI_ENDPOINT`` is preferred when explicitly set.
+* When absent, the endpoint is auto-derived from
+  ``AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`` — the CONTRIBUTING-stated promise
+  that ``agentops init`` already satisfies for the project endpoint.
+* When neither is available the error mentions both missing variables and
+  hints the ``execution: cloud`` escape hatch for the deployment-only case.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from agentops.pipeline.runtime import _model_config
+
+
+@pytest.fixture(autouse=True)
+def _clear_env(monkeypatch: pytest.MonkeyPatch) -> None:
+    for key in (
+        "AZURE_OPENAI_ENDPOINT",
+        "AZURE_OPENAI_DEPLOYMENT",
+        "AZURE_AI_MODEL_DEPLOYMENT_NAME",
+        "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT",
+        "AZURE_OPENAI_API_VERSION",
+    ):
+        monkeypatch.delenv(key, raising=False)
+
+
+def test_explicit_openai_endpoint_wins(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("AZURE_OPENAI_ENDPOINT", "https://explicit.openai.azure.com")
+    monkeypatch.setenv("AZURE_OPENAI_DEPLOYMENT", "gpt-4o-mini")
+    monkeypatch.setenv(
+        "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT",
+        "https://different.services.ai.azure.com/api/projects/p1",
+    )
+
+    cfg = _model_config()
+
+    assert cfg["azure_endpoint"] == "https://explicit.openai.azure.com"
+    assert cfg["azure_deployment"] == "gpt-4o-mini"
+
+
+def test_endpoint_auto_derived_from_project(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv(
+        "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT",
+        "https://ai-account-xyz.services.ai.azure.com/api/projects/proj-default",
+    )
+    monkeypatch.setenv("AZURE_OPENAI_DEPLOYMENT", "gpt-4o-mini")
+
+    cfg = _model_config()
+
+    assert cfg["azure_endpoint"] == "https://ai-account-xyz.services.ai.azure.com"
+    assert cfg["azure_deployment"] == "gpt-4o-mini"
+
+
+def test_missing_deployment_still_raises_with_hint(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv(
+        "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT",
+        "https://acct.services.ai.azure.com/api/projects/p1",
+    )
+
+    with pytest.raises(RuntimeError) as excinfo:
+        _model_config()
+
+    message = str(excinfo.value)
+    assert "AZURE_OPENAI_DEPLOYMENT" in message
+    assert "AZURE_OPENAI_ENDPOINT" not in message
+    # The hint nudges the user toward the cloud-execution escape hatch.
+    assert "execution: cloud" in message
+
+
+def test_missing_both_lists_endpoint_and_deployment(monkeypatch: pytest.MonkeyPatch) -> None:
+    with pytest.raises(RuntimeError) as excinfo:
+        _model_config()
+
+    message = str(excinfo.value)
+    assert "AZURE_OPENAI_ENDPOINT" in message
+    assert "AZURE_OPENAI_DEPLOYMENT" in message
+
+
+def test_default_api_version_pinned(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("AZURE_OPENAI_ENDPOINT", "https://x.openai.azure.com")
+    monkeypatch.setenv("AZURE_OPENAI_DEPLOYMENT", "gpt-4o-mini")
+
+    cfg = _model_config()
+
+    assert cfg["api_version"] == "2025-04-01-preview"
+
+
+def test_api_version_override(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("AZURE_OPENAI_ENDPOINT", "https://x.openai.azure.com")
+    monkeypatch.setenv("AZURE_OPENAI_DEPLOYMENT", "gpt-4o-mini")
+    monkeypatch.setenv("AZURE_OPENAI_API_VERSION", "2024-12-01-preview")
+
+    cfg = _model_config()
+
+    assert cfg["api_version"] == "2024-12-01-preview"