From b9154c8ba95767463fdd882f7e6476a8c10343de Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Mon, 1 Jun 2026 10:00:49 -0700 Subject: [PATCH 1/4] Add HyperDX OTel integration test path Wire an opt-in integration test against a live HyperDX endpoint via the OTLP-HTTP exporter, plus a "Production swap" section in example 03's docstring showing the same setup. The integration test (``tests/integration/test_otel_hyperdx_export.py``) is env-gated on ``HYPERDX_API_KEY`` + ``HYPERDX_OTLP_ENDPOINT`` and ``@pytest.mark.integration``; skipped by default. Verifies that the ``BatchSpanProcessor`` + ``OTLPSpanExporter`` pipeline drains cleanly within a 15s deadline. HyperDX-side acceptance is verified by the UI, not the test, because the OTel SDK swallows exporter errors silently. ``opentelemetry-exporter-otlp-proto-http`` lands in the dev dependency group only, capped ``<3`` for parity with the ``[otel]`` extras' SDK pins. Not promoted to a public extras group yet; revisit when more than one downstream user wants OTLP-HTTP export packaged. --- CHANGELOG.md | 4 + examples/03-observer-hooks/main.py | 37 ++++++++ pyproject.toml | 5 ++ tests/integration/test_otel_hyperdx_export.py | 90 +++++++++++++++++++ uv.lock | 2 + 5 files changed, 138 insertions(+) create mode 100644 tests/integration/test_otel_hyperdx_export.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ecfbcd..197eb30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). The ## [Unreleased] +### Added + +- **HyperDX OTel integration test path and "Production swap" docs in example 03.** `examples/03-observer-hooks/main.py`'s module docstring grows a "Production swap" section showing how to substitute the demo's `SimpleSpanProcessor` + `ConsoleSpanExporter` for `BatchSpanProcessor` + `OTLPSpanExporter` pointed at HyperDX (or any other OTLP-HTTP collector). A new opt-in integration test (`tests/integration/test_otel_hyperdx_export.py`, gated by `HYPERDX_API_KEY` + `HYPERDX_OTLP_ENDPOINT` env vars and `@pytest.mark.integration`) drives the same production export path end-to-end against a live endpoint. `opentelemetry-exporter-otlp-proto-http` lands as a dev-only dep; not promoted to a public extras group yet. + ### Changed (breaking) - **`OpenAIProvider.ready()` default probe flipped to `chat_completions`.** A new constructor kwarg `readiness_probe: Literal["models", "chat_completions", "both"]` selects which wire path `ready()` exercises; the default is now the chat-completions path (`POST /v1/chat/completions` with `max_tokens=1`), which actually exercises the inference path. The previous catalog-only behavior is still available as `readiness_probe="models"`, and `readiness_probe="both"` runs catalog then chat for the strongest signal. Motivation: OpenAI-compatible proxies (Bifrost and similar) can return 200 on `GET /v1/models` while rejecting `POST /v1/chat/completions`, leaving the catalog probe green while every real call fails. The new default surfaces that class of failure at preflight rather than at first inference. Non-200 chat-probe responses route through `classify_http_error`, so the canonical error categories (`provider_authentication`, `provider_unavailable`, `provider_invalid_model`, etc.) surface consistently. Callers that depended on the catalog-only behavior (cost-sensitive cloud setups where every `ready()` would now bill prompt tokens) can opt back in by passing `readiness_probe="models"`. diff --git a/examples/03-observer-hooks/main.py b/examples/03-observer-hooks/main.py index 9c8430c..3837f67 100644 --- a/examples/03-observer-hooks/main.py +++ b/examples/03-observer-hooks/main.py @@ -35,6 +35,43 @@ LLM_API_KEY=sk-... uv run python main.py "explain why NASA is returning to the moon with Artemis" (``--all-extras`` pulls in ``opentelemetry-sdk`` for the OTel observer.) + +**Production swap: real OTLP exporter (e.g. HyperDX).** + +The example wires ``OTelObserver`` to a ``SimpleSpanProcessor`` + +``ConsoleSpanExporter`` so every span prints to stdout. That is fine +for a short-lived demo and wrong for production: synchronous export +blocks each node boundary, and printing is not ingestion. For a real +backend (HyperDX, Honeycomb, Tempo, any OTLP-HTTP collector), swap to +``BatchSpanProcessor`` + ``OTLPSpanExporter`` pointing at your +collector and supplying its auth header. The HyperDX shape:: + + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + otel_observer = OTelObserver( + span_processor=BatchSpanProcessor( + OTLPSpanExporter( + endpoint="https://in-otel.hyperdx.io/v1/traces", + # HyperDX accepts the API key as a bare ``authorization`` + # value. Other collectors expect ``Bearer ``; + # check your destination's docs. The bracket-form + # ``os.environ[...]`` is intentional: unlike ``LLM_API_KEY`` + # (which permits None for unauthenticated local servers), + # a missing HyperDX key would silently send unauthenticated + # requests, so fail-loud at boot is the right shape. + headers={"authorization": os.environ["HYPERDX_API_KEY"]}, + ) + ), + resource=Resource.create({"service.name": "openarmature-demo-answers"}), + ) + +Same observer call surface; only the processor + exporter change. The +``OTLPSpanExporter`` lives in the ``opentelemetry-exporter-otlp-proto-http`` +package (not in ``[otel]`` extras yet; install it directly while OA +gauges demand). Remember to ``await otel_observer.force_flush()`` +before short-lived processes exit; ``BatchSpanProcessor`` ships in +the background and would otherwise drop the tail. """ from __future__ import annotations diff --git a/pyproject.toml b/pyproject.toml index 71e8b81..87b9830 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,11 @@ dev = [ "pyyaml>=6.0", "ruff>=0.5", "types-pyyaml", + # Used only by ``tests/integration/test_otel_hyperdx_export.py`` + # against a live HyperDX endpoint. Not promoted to a public extras + # group yet: one downstream user, one destination; revisit when + # multiple users want OTLP-HTTP export packaged. + "opentelemetry-exporter-otlp-proto-http>=1.27,<3", ] docs = [ "mkdocs>=1.6,<2", diff --git a/tests/integration/test_otel_hyperdx_export.py b/tests/integration/test_otel_hyperdx_export.py new file mode 100644 index 0000000..fb07119 --- /dev/null +++ b/tests/integration/test_otel_hyperdx_export.py @@ -0,0 +1,90 @@ +"""Integration test for OTel span export against a live HyperDX endpoint. + +Gated by the presence of ``HYPERDX_API_KEY`` + ``HYPERDX_OTLP_ENDPOINT`` +env vars. Skipped in CI and local runs that don't have credentials in +scope; runs end-to-end against HyperDX Cloud (or any other OTLP-HTTP +collector) when invoked from a shell with both env vars sourced. + +The test verifies the production export path the documentation +recommends (``BatchSpanProcessor`` + ``OTLPSpanExporter``) drains +cleanly from the local pipeline. The assertion is local-side: the +BatchSpanProcessor's ``force_flush`` succeeded within the deadline. +HyperDX-side acceptance (auth, payload accepted, span visible in the +UI) is verified by checking the HyperDX UI for a span named ``ping`` +under service ``openarmature-hyperdx-integration``; the OTel SDK +swallows exporter errors silently, so a local-side success does not +prove the collector received the spans. +""" + +from __future__ import annotations + +import os + +import pytest + +# Skip the entire module when credentials / endpoint aren't sourced. +# Avoids an ImportError cascade from the OTLP exporter if its env-var +# fallback also can't find a target. +pytestmark = pytest.mark.skipif( + not (os.environ.get("HYPERDX_API_KEY") and os.environ.get("HYPERDX_OTLP_ENDPOINT")), + reason="Requires HYPERDX_API_KEY + HYPERDX_OTLP_ENDPOINT (live HyperDX endpoint)", +) + + +@pytest.mark.integration +async def test_otel_observer_pipeline_drains_with_hyperdx_exporter() -> None: + """End-to-end: invoke a tiny graph under an OTelObserver wired to + the OTLPSpanExporter pointing at the configured HyperDX endpoint, + flush, and assert the local pipeline drained within the deadline. + """ + # Imports inside the function so the heavy OTLP-protobuf + # dependencies don't load when the module is collected and skipped + # under the default ``-m "not integration"`` pytest filter. + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + from openarmature.graph import END, GraphBuilder, State + from openarmature.observability.otel import OTelObserver + + # HyperDX accepts the API key as a bare ``authorization`` header + # value (no ``Bearer`` prefix). Other OTLP collectors that expect + # ``Bearer `` will need the caller to format the header + # themselves; this is the documented HyperDX shape. + exporter = OTLPSpanExporter( + endpoint=os.environ["HYPERDX_OTLP_ENDPOINT"], + headers={"authorization": os.environ["HYPERDX_API_KEY"]}, + ) + + observer = OTelObserver( + span_processor=BatchSpanProcessor(exporter), + resource=Resource.create({"service.name": "openarmature-hyperdx-integration"}), + ) + + class _PingState(State): + ping: bool = False + + async def _node(_s: _PingState) -> dict[str, bool]: + return {"ping": True} + + graph = GraphBuilder(_PingState).add_node("ping", _node).add_edge("ping", END).set_entry("ping").compile() + graph.attach_observer(observer) + + try: + final = await graph.invoke(_PingState()) + assert final.ping is True + + # Local-side assertion. ``BatchSpanProcessor.force_flush`` + # returns True when every registered processor finishes + # flushing within the timeout, False when any one times out. + # The OTel SDK swallows exporter-side errors (401s, schema + # rejections) silently, so a True here proves the pipeline + # drained but not that HyperDX accepted the payload; that + # confirmation is in the HyperDX UI. + flushed = observer.force_flush(timeout_ms=15_000) + assert flushed, "BatchSpanProcessor did not finish flushing within 15s" + finally: + # Releases the BatchSpanProcessor's background export thread; + # ``OTelObserver.shutdown`` is idempotent and calls + # ``_provider.shutdown`` under the hood. + observer.shutdown() diff --git a/uv.lock b/uv.lock index a2e4d1d..f021cf3 100644 --- a/uv.lock +++ b/uv.lock @@ -946,6 +946,7 @@ otel = [ [package.dev-dependencies] dev = [ + { name = "opentelemetry-exporter-otlp-proto-http" }, { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, @@ -985,6 +986,7 @@ provides-extras = ["otel", "langfuse"] [package.metadata.requires-dev] dev = [ + { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.27,<3" }, { name = "pre-commit", specifier = ">=3.7" }, { name = "pyright", specifier = ">=1.1.360" }, { name = "pytest", specifier = ">=8.2" }, From 324fcbfdfa3d959944249d9c4b9f5a3ca8858802 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Mon, 1 Jun 2026 10:27:53 -0700 Subject: [PATCH 2/4] Document HYPERDX_OTLP_ENDPOINT path requirement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OTLPSpanExporter uses the ``endpoint`` kwarg verbatim and does not append ``/v1/traces`` itself (that auto-append is only for the host-only ``OTEL_EXPORTER_OTLP_ENDPOINT`` env-var convention this test doesn't use). A host-only URL POSTs to ``/`` and HyperDX 404s, which the OTel SDK swallows silently while ``force_flush`` still returns True — the failure mode this test calls out. Note the path-suffix requirement explicitly in the module docstring and the pytestmark skip reason so future readers don't lose half an hour to a silent failure that wasn't surfaced. --- tests/integration/test_otel_hyperdx_export.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_otel_hyperdx_export.py b/tests/integration/test_otel_hyperdx_export.py index fb07119..9801129 100644 --- a/tests/integration/test_otel_hyperdx_export.py +++ b/tests/integration/test_otel_hyperdx_export.py @@ -5,6 +5,16 @@ scope; runs end-to-end against HyperDX Cloud (or any other OTLP-HTTP collector) when invoked from a shell with both env vars sourced. +``HYPERDX_OTLP_ENDPOINT`` MUST be the full traces-collector URL +including the ``/v1/traces`` path suffix, e.g.:: + + HYPERDX_OTLP_ENDPOINT=https://in-otel.hyperdx.io/v1/traces + +``OTLPSpanExporter`` uses the ``endpoint`` kwarg verbatim and does +not append the path itself (that auto-append only happens for the +``OTEL_EXPORTER_OTLP_ENDPOINT`` host-only convention this test does +not use). A host-only URL POSTs to ``/`` and HyperDX 404s. + The test verifies the production export path the documentation recommends (``BatchSpanProcessor`` + ``OTLPSpanExporter``) drains cleanly from the local pipeline. The assertion is local-side: the @@ -27,7 +37,10 @@ # fallback also can't find a target. pytestmark = pytest.mark.skipif( not (os.environ.get("HYPERDX_API_KEY") and os.environ.get("HYPERDX_OTLP_ENDPOINT")), - reason="Requires HYPERDX_API_KEY + HYPERDX_OTLP_ENDPOINT (live HyperDX endpoint)", + reason=( + "Requires HYPERDX_API_KEY + HYPERDX_OTLP_ENDPOINT (live HyperDX endpoint); " + "endpoint MUST include the /v1/traces path suffix" + ), ) From d4e3113bab7871e90c8ca362342d82aa329217c3 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Mon, 1 Jun 2026 10:32:58 -0700 Subject: [PATCH 3/4] Drain observer queue before flush; fix sync force_flush docs Two CoPilot findings on PR #110: 1. Integration test missed ``await graph.drain()`` between ``invoke()`` and ``observer.force_flush()``. ``invoke()`` returns when the graph reaches END, but observer events sit on a per- invocation queue until the background worker delivers them, so a span that hasn't seen its ``completed`` event yet is still open when ``force_flush`` runs. The live HyperDX run worked only because the queue drained inside the 15s flush window; under load the test could ship only ``started`` halves. 2. Example docstring suggested ``await otel_observer.force_flush()``. ``force_flush`` is synchronous (returns bool, wraps ``TracerProvider.force_flush``); the await would raise ``TypeError`` at runtime. Replaced with the canonical short-lived-process pattern: ``await graph.drain()`` then ``otel_observer.force_flush()`` (sync). --- examples/03-observer-hooks/main.py | 9 ++++++--- tests/integration/test_otel_hyperdx_export.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/examples/03-observer-hooks/main.py b/examples/03-observer-hooks/main.py index 3837f67..a60eb0d 100644 --- a/examples/03-observer-hooks/main.py +++ b/examples/03-observer-hooks/main.py @@ -69,9 +69,12 @@ Same observer call surface; only the processor + exporter change. The ``OTLPSpanExporter`` lives in the ``opentelemetry-exporter-otlp-proto-http`` package (not in ``[otel]`` extras yet; install it directly while OA -gauges demand). Remember to ``await otel_observer.force_flush()`` -before short-lived processes exit; ``BatchSpanProcessor`` ships in -the background and would otherwise drop the tail. +gauges demand). Before short-lived processes exit, call +``await graph.drain()`` (drains the observer's per-invocation event +queue so spans see their ``completed`` events) and then +``otel_observer.force_flush()`` (synchronous; pushes +``BatchSpanProcessor``'s tail through the exporter). The drain + flush +pair ensures the tail lands before teardown. """ from __future__ import annotations diff --git a/tests/integration/test_otel_hyperdx_export.py b/tests/integration/test_otel_hyperdx_export.py index 9801129..e788a18 100644 --- a/tests/integration/test_otel_hyperdx_export.py +++ b/tests/integration/test_otel_hyperdx_export.py @@ -87,6 +87,16 @@ async def _node(_s: _PingState) -> dict[str, bool]: final = await graph.invoke(_PingState()) assert final.ping is True + # ``invoke()`` returns when the graph reaches END but observer + # events sit on a per-invocation queue until the background + # worker drains them. Without ``drain()``, a span that hasn't + # yet seen its ``completed`` event is still open when + # ``force_flush`` runs, and the exporter would ship only the + # ``started`` half (or nothing at all). The short-lived-process + # pattern in ``docs/agent/non-obvious-shapes.md`` makes this + # explicit. + await graph.drain() + # Local-side assertion. ``BatchSpanProcessor.force_flush`` # returns True when every registered processor finishes # flushing within the timeout, False when any one times out. From 4f35f83427874e875f8f4342b3ad4b9b8b17417b Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Mon, 1 Jun 2026 10:44:26 -0700 Subject: [PATCH 4/4] Enforce HYPERDX_OTLP_ENDPOINT path at runtime The documented requirement (env var MUST include ``/v1/traces``) was only enforced through docs, not the test itself. Because the OTel SDK swallows exporter-side errors and ``force_flush`` returns True regardless of the HTTP response, a host-only URL would POST to ``/``, HyperDX would 404, and the test would still report green. This is the silent-failure mode the test's docstring already warned about and the one we hit on this PR's manual validation. Assert the endpoint ends with ``/v1/traces`` before constructing the exporter; the assert raises with a pointer to the expected shape if the env var is misconfigured. --- tests/integration/test_otel_hyperdx_export.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_otel_hyperdx_export.py b/tests/integration/test_otel_hyperdx_export.py index e788a18..f069c32 100644 --- a/tests/integration/test_otel_hyperdx_export.py +++ b/tests/integration/test_otel_hyperdx_export.py @@ -60,12 +60,24 @@ async def test_otel_observer_pipeline_drains_with_hyperdx_exporter() -> None: from openarmature.graph import END, GraphBuilder, State from openarmature.observability.otel import OTelObserver + # Enforce the documented endpoint shape at runtime. The + # ``OTLPSpanExporter`` uses the URL verbatim and does not append + # ``/v1/traces`` itself, so a host-only URL POSTs to ``/`` and + # HyperDX 404s; the SDK swallows that response and ``force_flush`` + # still returns True, which would mask a misconfigured env var + # behind a passing test. + endpoint = os.environ["HYPERDX_OTLP_ENDPOINT"] + assert endpoint.endswith("/v1/traces"), ( + f"HYPERDX_OTLP_ENDPOINT must end with /v1/traces (got {endpoint!r}); " + "OTLPSpanExporter uses the URL verbatim and does not append paths." + ) + # HyperDX accepts the API key as a bare ``authorization`` header # value (no ``Bearer`` prefix). Other OTLP collectors that expect # ``Bearer `` will need the caller to format the header # themselves; this is the documented HyperDX shape. exporter = OTLPSpanExporter( - endpoint=os.environ["HYPERDX_OTLP_ENDPOINT"], + endpoint=endpoint, headers={"authorization": os.environ["HYPERDX_API_KEY"]}, )