diff --git a/backend/apps/northbound_base_app.py b/backend/apps/northbound_base_app.py index f89a62c617..032a36678c 100644 --- a/backend/apps/northbound_base_app.py +++ b/backend/apps/northbound_base_app.py @@ -96,7 +96,7 @@ async def northbound_lifespan(_app): description="Northbound APIs for partners", version="1.0.0", cors_methods=["GET", "POST", "PUT", "DELETE"], - enable_monitoring=False, # Disable monitoring if not needed + enable_monitoring=True, lifespan=northbound_lifespan, ) if hasattr(northbound_app, "state"): diff --git a/backend/management/services/agent/runtime_client.py b/backend/management/services/agent/runtime_client.py index 157b2417fd..35977d0e17 100644 --- a/backend/management/services/agent/runtime_client.py +++ b/backend/management/services/agent/runtime_client.py @@ -1,7 +1,8 @@ # Copyright (c) 2025-2026 Huawei Technologies Co., Ltd. All rights reserved. """HTTP forwarding from northbound APIs to the runtime service.""" -from typing import AsyncIterator, Literal +from collections.abc import AsyncIterator +from typing import Literal import httpx from fastapi.responses import StreamingResponse @@ -13,9 +14,11 @@ RuntimeUpstreamError, ) from consts.model import AgentRequest -from management.services.permission.identity.authentication_helpers import generate_internal_runtime_jwt from infrastructure.http.client import create_httpx_client - +from management.services.permission.identity.authentication_helpers import ( + generate_internal_runtime_jwt, +) +from nexent.monitor.monitoring import get_monitoring_manager _HOP_BY_HOP_HEADERS = { "connection", @@ -53,7 +56,7 @@ def _authorization_headers(user_id: str, tenant_id: str) -> dict[str, str]: raise RuntimeServiceUnavailableError( "Internal runtime authentication is not configured" ) from exc - return {"Authorization": f"Bearer {token}"} + return {"Authorization": f"Bearer {token}", **get_monitoring_manager().get_trace_headers()} def dispatch_agent_evaluation_run( diff --git a/deploy/docker/compose/docker-compose.prod.yml b/deploy/docker/compose/docker-compose.prod.yml index 79d64de5f8..588fd3086f 100644 --- a/deploy/docker/compose/docker-compose.prod.yml +++ b/deploy/docker/compose/docker-compose.prod.yml @@ -191,6 +191,7 @@ services: UMASK: "0022" env_file: - ../../env/.env + - ../../env/monitoring.env user: root depends_on: nexent-elasticsearch: diff --git a/deploy/docker/compose/docker-compose.yml b/deploy/docker/compose/docker-compose.yml index 7974474a8b..7bd9431cee 100644 --- a/deploy/docker/compose/docker-compose.yml +++ b/deploy/docker/compose/docker-compose.yml @@ -213,6 +213,7 @@ services: UMASK: "0022" env_file: - ../../env/.env + - ../../env/monitoring.env user: root depends_on: nexent-elasticsearch: diff --git a/deploy/env/monitoring.env.example b/deploy/env/monitoring.env.example index 2f98941cde..5202888a34 100644 --- a/deploy/env/monitoring.env.example +++ b/deploy/env/monitoring.env.example @@ -16,7 +16,7 @@ OTEL_EXPORTER_OTLP_X_API_KEY= OTEL_EXPORTER_OTLP_LANGFUSE_INGESTION_VERSION= OTEL_EXPORTER_OTLP_METRICS_ENABLED=true MONITORING_INSTRUMENT_REQUESTS=false -MONITORING_FASTAPI_INCLUDED_URLS=/agent/run,/conversation/generate_title,/nb/v1/generate_title +MONITORING_FASTAPI_INCLUDED_URLS=/agent/run,/conversation/generate_title,/nb/,/agent/internal/northbound/ MONITORING_FASTAPI_EXCLUDED_URLS= MONITORING_FASTAPI_EXCLUDE_SPANS=receive,send # Comma-separated, case-sensitive exact names; whitespace and empty entries are ignored. diff --git a/deploy/k8s/helm/nexent/charts/nexent-northbound/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-northbound/templates/deployment.yaml index 34ee4ae602..41422f9b06 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-northbound/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-northbound/templates/deployment.yaml @@ -51,6 +51,8 @@ spec: name: nexent-config - secretRef: name: nexent-env + - secretRef: + name: nexent-monitoring-env - secretRef: name: nexent-secrets env: diff --git a/sdk/nexent/monitor/monitoring.py b/sdk/nexent/monitor/monitoring.py index e2171f60fc..17d18d60a8 100644 --- a/sdk/nexent/monitor/monitoring.py +++ b/sdk/nexent/monitor/monitoring.py @@ -29,7 +29,7 @@ from opentelemetry.sdk.trace import TracerProvider from opentelemetry.instrumentation.requests import RequestsInstrumentor from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor - from opentelemetry import trace, metrics + from opentelemetry import trace, metrics, propagate from opentelemetry.sdk.resources import Resource OPENTELEMETRY_AVAILABLE = True except ImportError: @@ -731,6 +731,18 @@ def tracer(self): """Get the tracer instance.""" return self._tracer + def get_trace_headers(self) -> Dict[str, str]: + """Propagate only W3C trace context across internal HTTP boundaries.""" + if not self.is_enabled: + return {} + carrier: Dict[str, str] = {} + try: + propagate.inject(carrier) + except Exception: + logger.debug("Unable to inject telemetry context", exc_info=True) + return {} + return {key: value for key, value in carrier.items() if key in {"traceparent", "tracestate"}} + def setup_fastapi_app(self, app) -> bool: """Setup monitoring for a FastAPI application.""" try: diff --git a/test-e2e/cases/NB-OTEL-001/case.yaml b/test-e2e/cases/NB-OTEL-001/case.yaml new file mode 100644 index 0000000000..638cc2cd5b --- /dev/null +++ b/test-e2e/cases/NB-OTEL-001/case.yaml @@ -0,0 +1,87 @@ +schema_version: '1.0' +stage: D3 +module: V5-BASELINE +case: + case_id: NB-OTEL-001 + feature_id: F-146 + business_rule_ids: + - F-146-RULE-03 + title: Northbound to Runtime complete real-model response trace + type: AGENT-IT + priority: P0 + status: active + automation: automated + objective: Prove actual W3C cross-service trace and complete HTTP response with + real model and Phoenix. + preconditions: + - Northbound/Runtime run tested source with telemetry and both routes included. + - Task-local NEXENT_TEST_CREDENTIALS contains Northbound URL/key, real Agent and + skill marker, Phoenix URL/key/project. + test_data: + variants: + - real_model_stream + - unauthenticated + steps: + - order: 1 + action: POST chat with unique W3C traceparent; consume complete SSE from real + model and bound skill. + expected: HTTP 200, actual skill marker, no error event. + - order: 2 + action: Read trace from authenticated Phoenix and inspect Northbound/Runtime HTTP, + Agent, model and tool spans. + expected: Shared trace and valid ancestry; ended Northbound span encloses normal + Runtime/model/tool execution; actual model input and tokens visible. + - order: 3 + action: POST chat without Authorization with another W3C traceparent and inspect + Phoenix. + expected: HTTP 401, ended Northbound span, no downstream invocation. + - order: 4 + action: Close another real request after response headers without consuming body; + inspect ended HTTP trace and poll actual persisted conversation. + expected: Northbound HTTP span ends; original persisted run completes exactly + one user/assistant pair with actual skill marker, without stop or resubmission. + expected_results: + - Actual final answer agrees with real tool/model execution. + - Normal Northbound HTTP duration encloses Runtime completion with shared trace + and ancestry. + - Unauthorized HTTP is traced without downstream execution. + - Disconnected HTTP span ends while the existing persisted-run continuation policy + remains unchanged. + forbidden_side_effects: + - Do not record Authorization or API keys in span attributes. + - Do not replace real model or product collaborators. + integration_boundary: + - Actual Northbound HTTP -> Runtime -> real Agent/model/tool -> Phoenix; no mocks. + observations: + - Sanitized trace IDs, ancestry, times, token usage and HTTP status. + cleanup: + - Retain task-owned conversations for trace review; preserve user data. + acceptance: + - id: AC-STREAM + description: Real complete streaming and cross-service trace. + steps: + - 1 + - 2 + expected_results: + - 1 + - 2 + forbidden_side_effects: + - 1 + - 2 + - id: AC-AUTH + description: Ended rejection span with no downstream execution. + steps: + - 3 + expected_results: + - 3 + forbidden_side_effects: + - 1 + - id: AC-DISCONNECT + description: HTTP cancellation ends its span while persisted Agent work continues. + steps: + - 4 + expected_results: + - 4 + forbidden_side_effects: + - 1 + - 2 diff --git a/test-e2e/cases/NB-OTEL-001/execution.yaml b/test-e2e/cases/NB-OTEL-001/execution.yaml new file mode 100644 index 0000000000..6abf5f87c8 --- /dev/null +++ b/test-e2e/cases/NB-OTEL-001/execution.yaml @@ -0,0 +1,14 @@ +schema_version: '1.0' +case_id: NB-OTEL-001 +implementations: +- framework: pytest + file: test.py + selector: NB-OTEL-001 +acceptance_bindings: + AC-STREAM: + - test_nb_otel_001_real_stream_and_trace_ancestry + AC-AUTH: + - test_nb_otel_001_unauthorized_trace_without_runtime + AC-DISCONNECT: + - test_nb_otel_001_disconnect_ends_http_but_keeps_persisted_run +timeout_seconds: 360 diff --git a/test-e2e/cases/NB-OTEL-001/test.py b/test-e2e/cases/NB-OTEL-001/test.py new file mode 100644 index 0000000000..31a6159b91 --- /dev/null +++ b/test-e2e/cases/NB-OTEL-001/test.py @@ -0,0 +1,156 @@ +# Copyright (c) 2025-2026 Huawei Technologies Co., Ltd. All rights reserved. +"""NB-OTEL-001: real Northbound/Runtime/model/tool trace via Phoenix.""" + +import json +import os +import time +from pathlib import Path +from urllib.parse import quote +from uuid import uuid4 + +import pytest +import requests + + +class PrivateConfig(dict): + """Keep test failure introspection from revealing credentials.""" + + def __repr__(self): + return '' + + +class SecretHeader(str): + def __repr__(self): + return '' + + +@pytest.fixture(scope='module') +def config(): + cfg = json.loads(Path(os.environ['NEXENT_TEST_CREDENTIALS']).read_text()) + required = ['northbound_url', 'northbound_api_key', 'agent_name', 'skill_name', + 'skill_marker', 'phoenix_url', 'phoenix_api_key', 'phoenix_project'] + assert all(cfg.get(key) for key in required), 'Incomplete private telemetry fixture' + return PrivateConfig(cfg) + + +def read_trace(cfg, trace_id, expected_route): + url = (cfg['phoenix_url'].rstrip('/') + '/v1/projects/' + + quote(cfg['phoenix_project'], safe='') + '/spans') + deadline = time.monotonic() + 45 + while True: + response = requests.get(url, params={'trace_id': trace_id, 'limit': 1000}, + headers={'Authorization': SecretHeader('Bearer ' + cfg['phoenix_api_key'])}, timeout=15) + if response.status_code == 404 and time.monotonic() < deadline: + time.sleep(1) + continue + assert response.status_code == 200, 'Authenticated Phoenix span query failed' + spans = response.json()['data'] + if any(expected_route in str(s['attributes'].get('http.route', '')) for s in spans): + break + assert time.monotonic() < deadline, 'Completed HTTP trace was not exported' + time.sleep(1) + serialized = json.dumps(spans) + assert cfg['northbound_api_key'] not in serialized and cfg['phoenix_api_key'] not in serialized + assert all(s['context']['trace_id'] == trace_id for s in spans) + assert all(s.get('end_time') for s in spans) + artifact_dir = os.environ.get('NEXENT_TEST_ARTIFACT_DIR') or os.environ.get('RESULT_DIR') + assert artifact_dir, 'Runner must provide a task-local artifact directory' + evidence = Path(artifact_dir) / 'runtime/northbound-otel' + evidence.mkdir(parents=True, exist_ok=True) + safe = [{'name': s['name'], 'context': s['context'], 'parent_id': s.get('parent_id'), + 'start_time': s['start_time'], 'end_time': s['end_time'], + 'status_code': s.get('status_code'), 'route': s['attributes'].get('http.route')} + for s in spans] + (evidence/f'{trace_id}.json').write_text(json.dumps(safe, indent=2)) + return spans + + +def chat(cfg, *, authenticated): + trace_id = uuid4().hex + headers = {'traceparent': '00-' + trace_id + '-1111111111111111-01'} + if authenticated: + headers['Authorization'] = SecretHeader('Bearer ' + cfg['northbound_api_key']) + body = {'agent_name': cfg['agent_name'], 'query': 'Read the bound skill ' + cfg['skill_name'] + + ' exactly once and report the actual guide marker. Do not invent tool results.'} + response = requests.post(cfg['northbound_url'].rstrip('/') + '/nb/v1/chat/run', + headers=headers, json=body, stream=True, timeout=180) + with response: + content = b''.join(response.iter_content(chunk_size=None)).decode() + return trace_id, response, content + + +def test_nb_otel_001_real_stream_and_trace_ancestry(config): + trace_id, response, content = chat(config, authenticated=True) + assert response.status_code == 200 + events = [json.loads(line[5:].strip()) for line in content.splitlines() + if line.startswith('data:') and line[5:].strip() != '[DONE]'] + assert not any(event.get('type') == 'error' for event in events) + finals = ''.join(str(e.get('content', '')) for e in events if e.get('type') == 'final_answer') + assert config['skill_marker'] in finals + spans = read_trace(config, trace_id, '/nb/v1/chat/run') + north = [s for s in spans if s['attributes'].get('http.route', '').endswith('/nb/v1/chat/run')] + runtime = [s for s in spans if '/agent/internal/northbound/run' in s['attributes'].get('http.route', '')] + assert len(north) == 1 and len(runtime) == 1 + root = north[0] + assert root['parent_id'] == '1111111111111111' + assert runtime[0]['parent_id'] == root['context']['span_id'] + assert root['start_time'] <= runtime[0]['start_time'] <= runtime[0]['end_time'] <= root['end_time'] + models = [s for s in spans if s['attributes'].get('llm.model_name')] + tools = [s for s in spans if s['name'] == 'agent.tool.read_skill_md'] + agents = [s for s in spans if s['name'] == 'agent.run'] + assert models and tools and agents + by_id = {s['context']['span_id']: s for s in spans} + for span in models + tools + agents: + assert root['start_time'] <= span['start_time'] <= span['end_time'] <= root['end_time'] + cursor = span + for _ in range(len(spans)): + if cursor['context']['span_id'] == root['context']['span_id']: + break + assert cursor.get('parent_id') in by_id, 'Broken trace ancestry' + cursor = by_id[cursor['parent_id']] + else: + pytest.fail('Trace parent cycle') + assert all(s['attributes'].get('input.truncated') is False for s in models) + assert all(s['attributes'].get('llm.token_count.total', 0) > 0 for s in models) + assert any(config['skill_marker'] in str(s['attributes'].get('input.value', '')) for s in models) + + +def test_nb_otel_001_unauthorized_trace_without_runtime(config): + trace_id, response, _content = chat(config, authenticated=False) + assert response.status_code == 401 + spans = read_trace(config, trace_id, '/nb/v1/chat/run') + north = [s for s in spans if s['attributes'].get('http.route', '').endswith('/nb/v1/chat/run')] + assert len(north) == 1 + attrs = north[0]['attributes'] + assert attrs.get('http.status_code', attrs.get('http.response.status_code')) == 401 + assert not any(s['name'].startswith('agent.') or s['attributes'].get('llm.model_name') + or '/agent/internal/' in s['attributes'].get('http.route', '') for s in spans) + + +pytestmark = [pytest.mark.case_id('NB-OTEL-001'), pytest.mark.stage('D3')] + + +def test_nb_otel_001_disconnect_ends_http_but_keeps_persisted_run(config): + trace_id = uuid4().hex + headers = {'Authorization': SecretHeader('Bearer ' + config['northbound_api_key']), + 'traceparent': '00-' + trace_id + '-1111111111111111-01'} + url = config['northbound_url'].rstrip('/') + with requests.post(url + '/nb/v1/chat/run', headers=headers, json={ + 'agent_name': config['agent_name'], 'query': 'Read the bound skill ' + config['skill_name'] + + ' exactly once and report its actual guide marker.'}, stream=True, timeout=180) as response: + assert response.status_code == 200 + conversation_id = int(response.headers['conversation_id']) + # Close after headers without consuming body or stopping the persisted producer. + spans = read_trace(config, trace_id, '/nb/v1/chat/run') + assert len([s for s in spans if s['attributes'].get('http.route', '').endswith('/nb/v1/chat/run')]) == 1 + deadline = time.monotonic() + 180 + while True: + history = requests.get(url + f'/nb/v1/conversations/{conversation_id}', headers={'Authorization': headers['Authorization']}, timeout=15) + assert history.status_code == 200 + messages = history.json()['data']['history'] + answers = [m['content'] for m in messages if m['role'] == 'assistant'] + if any(config['skill_marker'] in str(answer) for answer in answers): + break + assert time.monotonic() < deadline, 'Detached persisted run did not complete' + time.sleep(2) + assert len(answers) == 1 and sum(m['role'] == 'user' for m in messages) == 1 diff --git a/test-e2e/changes/bugs/NB-OTEL-20261111.yaml b/test-e2e/changes/bugs/NB-OTEL-20261111.yaml new file mode 100644 index 0000000000..d7203d3134 --- /dev/null +++ b/test-e2e/changes/bugs/NB-OTEL-20261111.yaml @@ -0,0 +1,53 @@ +schema_version: '1.0' +change: + change_id: NB-OTEL-20261111 + change_type: bugfix + bug_id: NB-OTEL-20261111 + title: Trace Northbound HTTP through complete response and Runtime + description: Enable ASGI instrumentation, supply deployment monitoring config and + propagate W3C context. + observed_behavior: Northbound monitoring is disabled; env/secret and Runtime traceparent + are missing. + expected_behavior: Included requests trace receipt through final send with Runtime + in the same trace. + escaped_stage: D3 + coverage_gap: + type: missing_boundary + description: Existing proxy and instrumentation UT/formal chat tests lack final + send and cross-service trace assertions. + feature_contract_changed: true + affected_features: + existing: + - F-146 + added: [] + modified: + - F-146 + retired: [] + affected_cases: + existing: [] + added: + - NB-OTEL-001 + modified: [] + retired: [] + traditional_ut: + decision: add + behaviors: + - ASGI final send lifetime, errors/cancellation and context isolation + - W3C propagation, disabled/missing telemetry and no baggage + - Docker/Helm monitoring parity and URL filters + test_selectors: + - test/sdk/monitor/test_http_response_lifecycle.py + - test/backend/services/test_runtime_proxy_telemetry.py + - test/backend/deploy/test_k8s_env_injection.py + reason: New unit boundaries and deployed D3 provide distinct missing lifecycle + and ancestry evidence. + notes: 'SPEC ../nexent-doc/nexent/codeagent-action-history-role/changes/v285-northbound-otel. + Independent Northbound tracing change split from PR4328; no tool-history or output-parser changes. No new D1: isolated UT and real D3 suffice; + existing disconnect/final-answer cases are adjacent regressions. Existing NB-CHAT-DISCONNECT-001 + and NB-CHAT-FINAL-ANSWER-001 were reviewed as unchanged wider business contracts, + not selected for full execution: relevant normal SSE and new-conversation disconnect + compatibility are exercised by NB-OTEL-001; existing/exhaustive answer-mode unit + suites remain adjacent regression. CI found seven stale Northbound env-injection/precedence + expectations in the existing Helm suite; align its monitored-service set, preserving + all ordering/platform/credential precedence assertions. Entire owning suite 24 + tests passed locally.' diff --git a/test-e2e/features/F-146/cases.md b/test-e2e/features/F-146/cases.md index 493d605cb5..aa9c9cd8a9 100644 --- a/test-e2e/features/F-146/cases.md +++ b/test-e2e/features/F-146/cases.md @@ -7,3 +7,4 @@ - D3 [AGT-065](../../cases/AGT-065/case.yaml) · active - D3 [AGT-066](../../cases/AGT-066/case.yaml) · active - D3 [NB-CHAT-DISCONNECT-001](../../cases/NB-CHAT-DISCONNECT-001/case.yaml) · active +- D3 [NB-OTEL-001](../../cases/NB-OTEL-001/case.yaml) · active diff --git a/test-e2e/features/F-146/feature.yaml b/test-e2e/features/F-146/feature.yaml index dd48a837d0..a691e31f91 100644 --- a/test-e2e/features/F-146/feature.yaml +++ b/test-e2e/features/F-146/feature.yaml @@ -6,12 +6,19 @@ feature: feature_id: F-146 name: new/existing conversation、agent_name/query/model override、SSE status: active - description: 'Northbound API / Chat Run: new/existing conversation、agent_name/query/model override、SSE' + description: 'Northbound API / Chat Run: new/existing conversation、agent_name/query/model + override、SSE' business_rules: - rule_id: F-146-RULE-01 description: new/existing conversation、agent_name/query/model override、SSE - rule_id: F-146-RULE-02 - description: Northbound chat defaults to one complete root final_answer; explicit final_answer_mode=delta retains incremental answer phases. Invalid modes return 422 before execution. + description: Northbound chat defaults to one complete root final_answer; explicit + final_answer_mode=delta retains incremental answer phases. Invalid modes return + 422 before execution. + - rule_id: F-146-RULE-03 + description: When telemetry is enabled and routes included, Northbound server + spans last through final ASGI send and propagate W3C context to Runtime/Agent; + errors or HTTP disconnect close the span without changing persisted-run continuation. required_test_stages: - D2 - D3 diff --git a/test/backend/deploy/test_k8s_env_injection.py b/test/backend/deploy/test_k8s_env_injection.py index 47579f7745..eeb04ae449 100644 --- a/test/backend/deploy/test_k8s_env_injection.py +++ b/test/backend/deploy/test_k8s_env_injection.py @@ -245,7 +245,7 @@ def test_default_helm_has_empty_env_secrets_and_service_specific_references(): for v in deployment["spec"]["template"]["spec"]["containers"][0].get("envFrom", [])] if name in BACKENDS: expected = [None, "nexent-env"] - if name in ("config", "runtime"): + if name in ("config", "runtime", "northbound"): expected.append("nexent-monitoring-env") assert references == expected + ["nexent-secrets"] else: @@ -380,8 +380,8 @@ def test_helm_composition_preserves_precedence_and_platform_settings(tmp_path, d for deployment in deployments: name = deployment["metadata"]["name"].removeprefix("nexent-") container, effective = container_environment(documents, deployment) - assert effective["SHARED"] == ("monitoring" if name in ("config", "runtime") else "root") - assert ("MONITOR_ONLY" in effective) == (name in ("config", "runtime")) + assert effective["SHARED"] == ("monitoring" if name in ("config", "runtime", "northbound") else "root") + assert ("MONITOR_ONLY" in effective) == (name in ("config", "runtime", "northbound")) assert effective["MINIO_ACCESS_KEY"] == "final-access" assert effective["EMPTY"] == "" assert effective["UNKNOWN_NEW_FEATURE"] == "enabled" diff --git a/test/backend/services/test_runtime_proxy_telemetry.py b/test/backend/services/test_runtime_proxy_telemetry.py new file mode 100644 index 0000000000..c69fa21000 --- /dev/null +++ b/test/backend/services/test_runtime_proxy_telemetry.py @@ -0,0 +1,82 @@ +# Copyright (c) 2025-2026 Huawei Technologies Co., Ltd. All rights reserved. +"""NB-AC02/04/05: runtime propagation and deployed monitoring configuration.""" + +import ast +from pathlib import Path + +import pytest +import yaml +from opentelemetry import context, propagate, trace +from opentelemetry.baggage import set_baggage +from opentelemetry.trace import NonRecordingSpan, SpanContext, TraceFlags, TraceState + +from management.services.agent import runtime_client +from nexent.monitor import monitoring + + +@pytest.fixture +def manager(monkeypatch): + instance = monitoring.get_monitoring_manager() + monkeypatch.setattr(instance, '_config', monitoring.MonitoringConfig(enable_telemetry=True)) + monkeypatch.setattr(runtime_client, 'generate_internal_runtime_jwt', lambda *_: 'internal-only-jwt') + return instance + + +def test_nb_ac002_internal_jwt_and_w3c_parent_survive_without_baggage(manager): + span_context = SpanContext(0x1234, 0x5678, True, TraceFlags(1), TraceState([('vendor', 'value')])) + ctx = trace.set_span_in_context(NonRecordingSpan(span_context)) + token = context.attach(set_baggage('private', 'never-forward', context=ctx)) + try: + headers = runtime_client._authorization_headers('user', 'tenant') + finally: + context.detach(token) + assert headers['Authorization'] == 'Bearer internal-only-jwt' + assert headers['traceparent'] == '00-00000000000000000000000000001234-0000000000005678-01' + assert headers['tracestate'] == 'vendor=value' + assert set(headers) == {'Authorization', 'traceparent', 'tracestate'} + parent = trace.get_current_span(propagate.extract(headers)).get_span_context() + assert parent.trace_id == span_context.trace_id and parent.span_id == span_context.span_id + assert manager.get_trace_headers() == {}, 'Context leaked beyond the original request' + + +@pytest.mark.parametrize('condition', ['disabled', 'missing_dependencies', 'injection_error']) +def test_nb_ac003_monitoring_failure_does_not_break_auth(manager, monkeypatch, condition): + if condition == 'disabled': + monkeypatch.setattr(manager, '_config', monitoring.MonitoringConfig(enable_telemetry=False)) + elif condition == 'missing_dependencies': + monkeypatch.setattr(monitoring, 'OPENTELEMETRY_AVAILABLE', False) + else: + def fail(*_args): + raise RuntimeError('controlled propagation failure') + monkeypatch.setattr(monitoring.propagate, 'inject', fail) + assert runtime_client._authorization_headers('user', 'tenant') == {'Authorization': 'Bearer internal-only-jwt'} + + +def test_nb_ac005_propagator_cannot_inject_authorization_or_baggage(manager, monkeypatch): + def inject(carrier): + carrier.update(traceparent='parent', Authorization='external-secret', baggage='private-data') + monkeypatch.setattr(monitoring.propagate, 'inject', inject) + assert runtime_client._authorization_headers('user', 'tenant') == { + 'Authorization': 'Bearer internal-only-jwt', 'traceparent': 'parent'} + + +def test_nb_ac004_northbound_enables_existing_monitoring_and_deployment_config(): + root = Path(__file__).resolve().parents[3] + tree = ast.parse((root/'backend/apps/northbound_base_app.py').read_text()) + calls = [node for node in ast.walk(tree) if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) and node.func.id == 'create_app'] + assert len(calls) == 1 + assert any(kw.arg == 'enable_monitoring' and ast.literal_eval(kw.value) is True for kw in calls[0].keywords) + for name in ['docker-compose.yml', 'docker-compose.prod.yml']: + services = yaml.safe_load((root/'deploy/docker/compose'/name).read_text())['services'] + assert '../../env/monitoring.env' in services['nexent-northbound']['env_file'] + helm = (root/'deploy/k8s/helm/nexent/charts/nexent-northbound/templates/deployment.yaml').read_text() + assert 'name: nexent-monitoring-env' in helm.split('envFrom:')[1].split(' env:')[0] + env = (root/'deploy/env/monitoring.env.example').read_text() + included = next(line.split('=', 1)[1] for line in env.splitlines() if line.startswith('MONITORING_FASTAPI_INCLUDED_URLS=')) + excluded = monitoring._build_fastapi_excluded_urls(included, '') + import re + assert not re.search(excluded, 'http://northbound:5013/api/nb/v1/chat/run') + assert not re.search(excluded, 'http://runtime:5014/api/agent/internal/northbound/run') + explicit = monitoring._build_fastapi_excluded_urls(included, '/nb/v1/chat/run') + assert any(re.search(pattern, 'http://northbound:5013/api/nb/v1/chat/run') for pattern in explicit.split(',')) diff --git a/test/sdk/monitor/_http_response_lifecycle_cases.py b/test/sdk/monitor/_http_response_lifecycle_cases.py new file mode 100644 index 0000000000..5ceb94a0ee --- /dev/null +++ b/test/sdk/monitor/_http_response_lifecycle_cases.py @@ -0,0 +1,162 @@ +# Copyright (c) 2025-2026 Huawei Technologies Co., Ltd. All rights reserved. +"""NB-AC01/03: real ASGI instrumentation covers the complete response send.""" + +import asyncio + +import pytest +from fastapi import FastAPI +from fastapi.responses import JSONResponse, StreamingResponse +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import SpanKind, StatusCode + +from nexent.monitor.monitoring import MonitoringConfig, get_monitoring_manager + + +@pytest.fixture +def instrument(monkeypatch): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + monkeypatch.setattr(trace, 'get_tracer_provider', lambda: provider) + manager = get_monitoring_manager() + monkeypatch.setattr(manager, '_config', MonitoringConfig(enable_telemetry=True)) + yield manager, exporter + provider.shutdown() + + +def scope(path='/nb/v1/chat/run', headers=None): + return {'type': 'http', 'asgi': {'version': '3.0', 'spec_version': '2.4'}, + 'http_version': '1.1', 'method': 'POST', 'scheme': 'http', 'path': path, + 'raw_path': path.encode(), 'query_string': b'', 'root_path': '', + 'headers': headers or [], 'server': ('northbound', 5013), 'client': ('client', 1234)} + + +async def receive(): + await asyncio.Event().wait() + + +def server_spans(exporter): + return [span for span in exporter.get_finished_spans() if span.kind == SpanKind.SERVER] + + +@pytest.mark.asyncio +@pytest.mark.parametrize('outcome', ['complete', 'send_error', 'cancel']) +async def test_nb_ac001_span_stays_open_until_last_send(instrument, outcome): + manager, exporter = instrument + app = FastAPI() + last_send = asyncio.Event() + release = asyncio.Event() + chunks = [] + + @app.post('/nb/v1/chat/run') + async def run(): + async def body(): + yield b'data: first\n\n' + yield b'data: last\n\n' + return StreamingResponse(body(), media_type='text/event-stream') + + assert manager.setup_fastapi_app(app) + + async def send(message): + chunks.append(message) + if message['type'] == 'http.response.body' and not message.get('more_body', False): + last_send.set() + await release.wait() + if outcome == 'send_error': + raise RuntimeError('controlled transport failure') + + task = asyncio.create_task(app(scope(), receive, send)) + await asyncio.wait_for(last_send.wait(), 3) + assert not server_spans(exporter), 'Server span ended before last send completed' + assert b''.join(m.get('body', b'') for m in chunks) == b'data: first\n\ndata: last\n\n' + if outcome == 'cancel': + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + release.set() + if outcome == 'send_error': + with pytest.raises(RuntimeError, match='controlled transport failure'): + await task + else: + await task + spans = server_spans(exporter) + assert len(spans) == 1 and spans[0].end_time >= spans[0].start_time + if outcome == 'send_error': + assert spans[0].status.status_code == StatusCode.ERROR + + +@pytest.mark.asyncio +@pytest.mark.parametrize('status', [200, 401, 503]) +async def test_nb_ac003_nonstream_and_http_errors_end_span(instrument, status): + manager, exporter = instrument + app = FastAPI() + + @app.post('/nb/v1/chat/run') + async def run(): + return JSONResponse({'result': 'unchanged'}, status_code=status) + + assert manager.setup_fastapi_app(app) + messages = [] + + async def send(message): + messages.append(message) + + await app(scope(), receive, send) + spans = server_spans(exporter) + assert len(spans) == 1 + assert spans[0].attributes.get('http.status_code', spans[0].attributes.get('http.response.status_code')) == status + assert messages[0]['status'] == status + assert b'unchanged' in messages[-1]['body'] + + +@pytest.mark.asyncio +async def test_nb_ac002_concurrent_requests_keep_separate_trace_context(instrument): + manager, exporter = instrument + app = FastAPI() + entered = 0 + both = asyncio.Event() + + @app.post('/nb/v1/chat/run') + async def run(): + nonlocal entered + entered += 1 + if entered == 2: + both.set() + await both.wait() + return JSONResponse({'ok': True}) + + assert manager.setup_fastapi_app(app) + + async def send(_message): + pass + + ids = ['1' * 32, '2' * 32] + await asyncio.gather(*(app(scope(headers=[(b'traceparent', f'00-{tid}-1111111111111111-01'.encode())]), + receive, send) for tid in ids)) + spans = server_spans(exporter) + assert {f'{span.context.trace_id:032x}' for span in spans} == set(ids) + assert len({span.context.span_id for span in spans}) == 2 + + +@pytest.mark.asyncio +async def test_nb_ac003_disabled_monitoring_keeps_response_without_spans(instrument, monkeypatch): + manager, exporter = instrument + monkeypatch.setattr(manager, '_config', MonitoringConfig(enable_telemetry=False)) + app = FastAPI() + + @app.post('/nb/v1/chat/run') + async def run(): + return JSONResponse({'ok': True}) + + assert manager.setup_fastapi_app(app) is False + messages = [] + + async def send(message): + messages.append(message) + + await app(scope(), receive, send) + assert messages[0]['status'] == 200 and not exporter.get_finished_spans() diff --git a/test/sdk/monitor/test_http_response_lifecycle.py b/test/sdk/monitor/test_http_response_lifecycle.py new file mode 100644 index 0000000000..2c3b245caf --- /dev/null +++ b/test/sdk/monitor/test_http_response_lifecycle.py @@ -0,0 +1,18 @@ +# Copyright (c) 2025-2026 Huawei Technologies Co., Ltd. All rights reserved. +"""Run real OTel ASGI cases outside the global legacy OTel test stubs.""" + +import os +import subprocess +import sys +from pathlib import Path + + +def test_nb_ac001_real_asgi_response_lifecycle(tmp_path): + root = Path(__file__).resolve().parents[3] + cases = Path(__file__).with_name('_http_response_lifecycle_cases.py') + env = {**os.environ, 'PYTHONPATH': str(root/'sdk')} + result = subprocess.run([sys.executable, '-m', 'pytest', str(cases), + '--noconftest', '--basetemp', str(tmp_path/'child'), '-q'], + cwd=root, env=env, capture_output=True, text=True, timeout=60, check=False) + assert result.returncode == 0, result.stdout + result.stderr + assert '8 passed' in result.stdout