diff --git a/Cargo.lock b/Cargo.lock index 9ed9b67d66..145cbd0fcb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5208,11 +5208,13 @@ dependencies = [ "openshell-isolation-interface", "openshell-ocsf", "openshell-otel", + "openshell-otel-test-support", "openshell-policy", "openshell-policy-schema", "openshell-sandbox-backend", "openshell-supervisor-middleware", "openshell-supervisor-middleware-builtins", + "opentelemetry-proto", "opentelemetry_sdk", "prost-types", "rcgen", diff --git a/crates/openshell-ocsf/schemas/ocsf/README.md b/crates/openshell-ocsf/schemas/ocsf/README.md index 2ec7b7ab0d..9f172f0e00 100644 --- a/crates/openshell-ocsf/schemas/ocsf/README.md +++ b/crates/openshell-ocsf/schemas/ocsf/README.md @@ -1,11 +1,12 @@ # Vendored OCSF Schemas These schemas are vendored from the [OCSF Schema Server](https://schema.ocsf.io/) -for offline test validation. +for offline test validation, with the OpenShell Trace-profile applicability +extension described below. ## Version -- **OCSF v1.8.0** — fetched from `https://schema.ocsf.io/api/1.8.0/` +- OCSF v1.8.0, fetched from `https://schema.ocsf.io/api/1.8.0/` ## Contents @@ -22,23 +23,39 @@ for offline test validation. - `authentication` [3002] - `base_event` [0] -### Objects (23) +### Objects (26) - `metadata`, `network_endpoint`, `network_proxy`, `process`, `actor` - `device`, `container`, `product`, `firewall_rule`, `finding_info` - `evidences`, `http_request`, `http_response`, `url`, `attack` -- `remediation`, `connection_info`, `ai_model`, `api`, `user`, `service`, `os`, - `resource_details` +- `remediation`, `connection_info`, `ai_model`, `api`, `user`, `service`, `os` +- `resource_details`, `trace`, `span`, `key_value_object` -### Profiles (1) +### Profiles (2) -- `ai_operation` +- `ai_operation`, `trace` + +Official OCSF 1.8.0 includes the Trace profile on +[HTTP Activity](https://github.com/ocsf/ocsf-schema/blob/v1.8.0/events/network/http_activity.json) +and [API Activity](https://github.com/ocsf/ocsf-schema/blob/v1.8.0/events/application/api_activity.json). +OpenShell extends this optional profile to `network_activity`, +`detection_finding`, `authentication`, `base_event`, `ssh_activity`, +`process_activity`, `application_lifecycle`, and `device_config_state_change`. +All ten vendored class schemas therefore include the profile's `trace` attribute +and list `trace` in `profiles`. + +This is an OpenShell extension beyond official 1.8.0 profile applicability. +Validation against this catalog checks that extension, not conformance with +unmodified official class definitions. Consumers that enforce official profile +applicability may reject correlated records from the eight additional classes. +The attribute is recommended, not required. The `trace`, `span`, `service`, and +`key_value_object` schemas are vendored unchanged. Events contain only `trace.uid`. ## Updating To update to a new OCSF version: -```bash +```shell VERSION=1.8.0 for class in network_activity http_activity ssh_activity process_activity \ @@ -51,11 +68,32 @@ done for object in metadata network_endpoint network_proxy process actor device \ container product firewall_rule finding_info evidences \ http_request http_response url attack remediation connection_info \ - ai_model api user service os; do + ai_model api user service os resource_details trace span key_value_object; do curl -s "https://schema.ocsf.io/api/${VERSION}/objects/${object}" \ | python3 -m json.tool > "objects/${object}.json" done +for profile in ai_operation trace; do + curl -s "https://schema.ocsf.io/api/${VERSION}/profiles/${profile}" \ + | python3 -m json.tool > "profiles/${profile}.json" +done + +# Compose Trace on every class after refreshing the official schemas. +uv run python - <<'PY' +import copy +import json +from pathlib import Path + +attribute = json.loads(Path("profiles/trace.json").read_text())["attributes"]["trace"] +attribute["profiles"] = ["trace"] +for path in Path("classes").glob("*.json"): + schema = json.loads(path.read_text()) + schema["attributes"]["trace"] = copy.deepcopy(attribute) + if "trace" not in schema.setdefault("profiles", []): + schema["profiles"].append("trace") + path.write_text(json.dumps(schema, indent=4) + "\n") +PY + echo "${VERSION}" > VERSION ``` diff --git a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/application_lifecycle.json b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/application_lifecycle.json index 7bf04ea1fa..fc98c4cd25 100644 --- a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/application_lifecycle.json +++ b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/application_lifecycle.json @@ -901,6 +901,18 @@ "security_control" ], "type_name": "Integer" + }, + "trace": { + "type": "object_t", + "description": "The trace object contains information about distributed traces which are critical to observability and describe how requests move through a system, capturing each step's timing and status.", + "group": "primary", + "requirement": "recommended", + "object_type": "trace", + "object_name": "Trace", + "caption": "Trace", + "profiles": [ + "trace" + ] } }, "name": "application_lifecycle", @@ -924,7 +936,8 @@ "linux/linux_users", "macos/macos_users", "osint", - "security_control" + "security_control", + "trace" ], "category_name": "Application Activity", "category_uid": 6 diff --git a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/authentication.json b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/authentication.json index afd15156f7..e2fd36b0d6 100644 --- a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/authentication.json +++ b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/authentication.json @@ -1221,6 +1221,18 @@ "object_type": "process", "object_name": "Process", "caption": "Logon Process" + }, + "trace": { + "type": "object_t", + "description": "The trace object contains information about distributed traces which are critical to observability and describe how requests move through a system, capturing each step's timing and status.", + "group": "primary", + "requirement": "recommended", + "object_type": "trace", + "object_name": "Trace", + "caption": "Trace", + "profiles": [ + "trace" + ] } }, "name": "authentication", @@ -1249,7 +1261,8 @@ "linux/linux_users", "macos/macos_users", "osint", - "security_control" + "security_control", + "trace" ], "caption": "Authentication", "associations": { diff --git a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/base_event.json b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/base_event.json index 14eee209b5..fbcbe618c8 100644 --- a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/base_event.json +++ b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/base_event.json @@ -826,6 +826,18 @@ "security_control" ], "type_name": "Integer" + }, + "trace": { + "type": "object_t", + "description": "The trace object contains information about distributed traces which are critical to observability and describe how requests move through a system, capturing each step's timing and status.", + "group": "primary", + "requirement": "recommended", + "object_type": "trace", + "object_name": "Trace", + "caption": "Trace", + "profiles": [ + "trace" + ] } }, "name": "base_event", @@ -842,7 +854,8 @@ "linux/linux_users", "macos/macos_users", "osint", - "security_control" + "security_control", + "trace" ], "category_uid": 0 } diff --git a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/detection_finding.json b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/detection_finding.json index f0a1adc9d7..57e7912ca3 100644 --- a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/detection_finding.json +++ b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/detection_finding.json @@ -1185,6 +1185,18 @@ "incident" ], "object_name": "User" + }, + "trace": { + "type": "object_t", + "description": "The trace object contains information about distributed traces which are critical to observability and describe how requests move through a system, capturing each step's timing and status.", + "group": "primary", + "requirement": "recommended", + "object_type": "trace", + "object_name": "Trace", + "caption": "Trace", + "profiles": [ + "trace" + ] } }, "name": "detection_finding", @@ -1203,7 +1215,8 @@ "linux/linux_users", "macos/macos_users", "osint", - "security_control" + "security_control", + "trace" ], "category_name": "Findings", "category_uid": 2 diff --git a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/device_config_state_change.json b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/device_config_state_change.json index 4ad3746fe6..fbeb83b423 100644 --- a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/device_config_state_change.json +++ b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/device_config_state_change.json @@ -963,6 +963,18 @@ "security_control" ], "type_name": "Integer" + }, + "trace": { + "type": "object_t", + "description": "The trace object contains information about distributed traces which are critical to observability and describe how requests move through a system, capturing each step's timing and status.", + "group": "primary", + "requirement": "recommended", + "object_type": "trace", + "object_name": "Trace", + "caption": "Trace", + "profiles": [ + "trace" + ] } }, "name": "device_config_state_change", @@ -980,7 +992,8 @@ "linux/linux_users", "macos/macos_users", "osint", - "security_control" + "security_control", + "trace" ], "category_name": "Discovery", "category_uid": 5 diff --git a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/network_activity.json b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/network_activity.json index 88cf930542..9f779d03ea 100644 --- a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/network_activity.json +++ b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/network_activity.json @@ -1144,6 +1144,18 @@ "security_control" ], "type_name": "Integer" + }, + "trace": { + "type": "object_t", + "description": "The trace object contains information about distributed traces which are critical to observability and describe how requests move through a system, capturing each step's timing and status.", + "group": "primary", + "requirement": "recommended", + "object_type": "trace", + "object_name": "Trace", + "caption": "Trace", + "profiles": [ + "trace" + ] } }, "name": "network_activity", @@ -1175,7 +1187,8 @@ "macos/macos_users", "network_proxy", "osint", - "security_control" + "security_control", + "trace" ], "category_name": "Network Activity", "category_uid": 4 diff --git a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/process_activity.json b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/process_activity.json index 1758412e7a..de21b2fb94 100644 --- a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/process_activity.json +++ b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/process_activity.json @@ -1033,6 +1033,18 @@ "security_control" ], "type_name": "Integer" + }, + "trace": { + "type": "object_t", + "description": "The trace object contains information about distributed traces which are critical to observability and describe how requests move through a system, capturing each step's timing and status.", + "group": "primary", + "requirement": "recommended", + "object_type": "trace", + "object_name": "Trace", + "caption": "Trace", + "profiles": [ + "trace" + ] } }, "name": "process_activity", @@ -1057,7 +1069,8 @@ "linux/linux_users", "macos/macos_users", "osint", - "security_control" + "security_control", + "trace" ], "associations": { "device": [ diff --git a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/ssh_activity.json b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/ssh_activity.json index 6175425740..b47b4ab6a1 100644 --- a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/ssh_activity.json +++ b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/classes/ssh_activity.json @@ -1213,6 +1213,18 @@ "security_control" ], "type_name": "Integer" + }, + "trace": { + "type": "object_t", + "description": "The trace object contains information about distributed traces which are critical to observability and describe how requests move through a system, capturing each step's timing and status.", + "group": "primary", + "requirement": "recommended", + "object_type": "trace", + "object_name": "Trace", + "caption": "Trace", + "profiles": [ + "trace" + ] } }, "name": "ssh_activity", @@ -1244,7 +1256,8 @@ "macos/macos_users", "network_proxy", "osint", - "security_control" + "security_control", + "trace" ], "category_name": "Network Activity", "category_uid": 4 diff --git a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/objects/key_value_object.json b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/objects/key_value_object.json new file mode 100644 index 0000000000..da3fdc4144 --- /dev/null +++ b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/objects/key_value_object.json @@ -0,0 +1,36 @@ +{ + "attributes": { + "name": { + "type": "string_t", + "description": "The name of the key.", + "requirement": "required", + "caption": "Name", + "type_name": "String" + }, + "value": { + "type": "string_t", + "description": "The value associated to the key.", + "requirement": "recommended", + "caption": "Value", + "type_name": "String" + }, + "values": { + "type": "string_t", + "description": "Optional, the values associated to the key. You can populate this attribute, when you have multiple values for the same key.", + "is_array": true, + "requirement": "recommended", + "caption": "Values", + "type_name": "String" + } + }, + "name": "key_value_object", + "description": "A generic object allowing to define a {key:value} pair.", + "extends": "object", + "constraints": { + "at_least_one": [ + "value", + "values" + ] + }, + "caption": "Key:Value object" +} diff --git a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/objects/span.json b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/objects/span.json new file mode 100644 index 0000000000..fb560f0dee --- /dev/null +++ b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/objects/span.json @@ -0,0 +1,95 @@ +{ + "attributes": { + "message": { + "type": "string_t", + "description": "The message in a span (often referred to as a span event) serves as a way to record significant moments or occurrences during the span's lifecycle. This content typically manifests as log entries, annotations, or semi-structured events as a string, providing additional granularity and context about what happens at specific points during the execution of an operation.", + "requirement": "optional", + "caption": "Message", + "type_name": "String" + }, + "service": { + "type": "object_t", + "description": "Identifies the service or component that generates the span, helping trace its path through the distributed system.", + "requirement": "optional", + "object_type": "service", + "object_name": "Service", + "caption": "Service" + }, + "uid": { + "type": "string_t", + "description": "The unique identifier for the span, used in distributed systems and microservices architectures to track and correlate requests across different components of an application. It enables tracing the flow of a request through various services.", + "requirement": "required", + "caption": "Unique ID", + "type_name": "String" + }, + "operation": { + "type": "string_t", + "description": "Describes an action performed in a span, such as API requests, database queries, or computations.", + "requirement": "optional", + "caption": "Operation", + "type_name": "String" + }, + "start_time": { + "type": "timestamp_t", + "description": "The start timestamp of the span, essential for identifying latency and performance bottlenecks. This timestamp is normalized across the observability system, ensuring consistency even when events occur across distributed services with potentially unsynchronized clocks. By using normalized time, observability tools can provide accurate, uniform measurements of operation performance and latency, regardless of where or when the events actually occur.", + "requirement": "required", + "caption": "Start Time", + "type_name": "Timestamp" + }, + "duration": { + "type": "long_t", + "description": "The total time, in milliseconds, that the span represents, calculated as the difference between start_time and end_time. It reflects the operation's performance and latency, independent of event timestamps, and accounts for normalized times used by observability tools to ensure consistency across distributed systems.", + "requirement": "optional", + "caption": "Duration Milliseconds", + "type_name": "Long" + }, + "end_time": { + "type": "timestamp_t", + "description": "The end timestamp of the span, essential for identifying latency and performance bottlenecks. Like the start time, this timestamp is normalized across the observability system to ensure consistency, even when events are recorded across distributed services with unsynchronized clocks. Normalized time allows for accurate duration calculations and helps observability tools track performance across services, regardless of the individual system time settings.", + "requirement": "required", + "caption": "End Time", + "type_name": "Timestamp" + }, + "start_time_dt": { + "type": "datetime_t", + "description": "The start timestamp of the span, essential for identifying latency and performance bottlenecks. This timestamp is normalized across the observability system, ensuring consistency even when events occur across distributed services with potentially unsynchronized clocks. By using normalized time, observability tools can provide accurate, uniform measurements of operation performance and latency, regardless of where or when the events actually occur.", + "requirement": "optional", + "profiles": [ + "datetime" + ], + "caption": "Start Time", + "type_name": "Datetime" + }, + "end_time_dt": { + "type": "datetime_t", + "description": "The end timestamp of the span, essential for identifying latency and performance bottlenecks. Like the start time, this timestamp is normalized across the observability system to ensure consistency, even when events are recorded across distributed services with unsynchronized clocks. Normalized time allows for accurate duration calculations and helps observability tools track performance across services, regardless of the individual system time settings.", + "requirement": "optional", + "profiles": [ + "datetime" + ], + "caption": "End Time", + "type_name": "Datetime" + }, + "parent_uid": { + "type": "string_t", + "description": "The ID of the parent span for this span object, establishing its relationship in the trace hierarchy.", + "requirement": "optional", + "caption": "Parent Unique ID", + "type_name": "String" + }, + "status_code": { + "type": "string_t", + "description": "Indicates the outcome of the operation in the span, such as success, failure, or error. Issues in a span typically refer to problems such as failed operations, timeouts, service unavailability, or errors in processing that can negatively impact the performance or reliability of the system. Tracking the `status_code` helps pinpoint these issues, enabling quicker identification and resolution of system inefficiencies or faults.", + "requirement": "optional", + "caption": "Status Code", + "type_name": "String" + } + }, + "name": "span", + "description": "Represents a single unit of work or operation within a distributed trace. A span typically tracks the execution of a request across a service, capturing important details such as the operation, timestamps, and status. Spans help break down the overall trace into smaller, manageable parts, enabling detailed analysis of the performance and behavior of specific operations within the system. They are crucial for understanding latency, dependencies, and bottlenecks in complex distributed systems.", + "extends": "object", + "profiles": [ + "datetime" + ], + "caption": "Span" +} diff --git a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/objects/trace.json b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/objects/trace.json new file mode 100644 index 0000000000..2ce4db5f3a --- /dev/null +++ b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/objects/trace.json @@ -0,0 +1,83 @@ +{ + "attributes": { + "flags": { + "type": "string_t", + "description": "The flags associated with the trace, used to indicate specific properties or behaviors, such as whether the trace is sampled or if it has special handling. Flags help control how traces are processed, logged, and analyzed, providing valuable context for tracing and observability tools in identifying trace characteristics or specific tracking requirements.", + "is_array": true, + "requirement": "optional", + "caption": "Flags", + "type_name": "String" + }, + "service": { + "type": "object_t", + "description": "Identifies the service or component generating the trace, helping to track and correlate the flow of requests through various parts of a distributed system. This information is essential for understanding the role and performance of specific services within the broader context of system operations and for diagnosing issues across different components.", + "requirement": "optional", + "object_type": "service", + "object_name": "Service", + "caption": "Service" + }, + "uid": { + "type": "string_t", + "description": "The unique identifier of the trace used in distributed systems and microservices architecture to track and correlate requests across various components of an application.", + "requirement": "required", + "caption": "Unique ID", + "type_name": "String" + }, + "span": { + "type": "object_t", + "description": "Represents a single unit of work or operation within a distributed trace. A span typically tracks the execution of a request across a service, capturing important details such as the operation, timestamps, and status. Spans help break down the overall trace into smaller, manageable parts, enabling detailed analysis of the performance and behavior of specific operations within the system. They are crucial for understanding latency, dependencies, and bottlenecks in complex distributed systems.", + "requirement": "optional", + "object_type": "span", + "object_name": "Span", + "caption": "Span" + }, + "start_time": { + "type": "timestamp_t", + "description": "The start timestamp of the trace, essential for identifying latency and performance bottlenecks. Like the end time, this timestamp is normalized across the trace system to ensure consistency, even when events are recorded across distributed services with unsynchronized clocks. Normalized time enables accurate trace duration calculations and helps observability tools track performance across services, regardless of the individual system time settings.", + "requirement": "optional", + "caption": "Start Time", + "type_name": "Timestamp" + }, + "duration": { + "type": "long_t", + "description": "The total time, in milliseconds, that the trace covers, calculated as the difference between start_time and end_time. This duration helps assess the overall performance of a request as it travels across various services, and is essential for identifying latency and potential bottlenecks within the distributed system. The trace duration may differ from individual span durations due to the propagation and processing times of the trace as it spans multiple components.", + "requirement": "optional", + "caption": "Duration Milliseconds", + "type_name": "Long" + }, + "end_time": { + "type": "timestamp_t", + "description": "The end timestamp of the trace, essential for identifying latency and performance bottlenecks. Like the start time, this timestamp is normalized across the trace system to ensure consistency, even when events are recorded across distributed services with unsynchronized clocks. Normalized time allows for accurate trace duration calculations and helps observability tools track overall performance across services, regardless of the individual system time settings.", + "requirement": "optional", + "caption": "End Time", + "type_name": "Timestamp" + }, + "start_time_dt": { + "type": "datetime_t", + "description": "The start timestamp of the trace, essential for identifying latency and performance bottlenecks. Like the end time, this timestamp is normalized across the trace system to ensure consistency, even when events are recorded across distributed services with unsynchronized clocks. Normalized time enables accurate trace duration calculations and helps observability tools track performance across services, regardless of the individual system time settings.", + "requirement": "optional", + "profiles": [ + "datetime" + ], + "caption": "Start Time", + "type_name": "Datetime" + }, + "end_time_dt": { + "type": "datetime_t", + "description": "The end timestamp of the trace, essential for identifying latency and performance bottlenecks. Like the start time, this timestamp is normalized across the trace system to ensure consistency, even when events are recorded across distributed services with unsynchronized clocks. Normalized time allows for accurate trace duration calculations and helps observability tools track overall performance across services, regardless of the individual system time settings.", + "requirement": "optional", + "profiles": [ + "datetime" + ], + "caption": "End Time", + "type_name": "Datetime" + } + }, + "name": "trace", + "description": "The trace object contains information about a distributed trace, which is crucial for observability. Traces are made up of one or more spans, which are individual units of work in application activity. Traces track the journey of a request as it moves through various services in a system, capturing key details like timing, status, and dependencies at each step. Traces provide insights into system performance, helping to identify latency, bottlenecks, and issues in complex, distributed environments.", + "extends": "object", + "profiles": [ + "datetime" + ], + "caption": "Trace" +} diff --git a/crates/openshell-ocsf/schemas/ocsf/v1.8.0/profiles/trace.json b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/profiles/trace.json new file mode 100644 index 0000000000..10132c9c50 --- /dev/null +++ b/crates/openshell-ocsf/schemas/ocsf/v1.8.0/profiles/trace.json @@ -0,0 +1,20 @@ +{ + "attributes": { + "trace": { + "type": "object_t", + "description": "The trace object contains information about distributed traces which are critical to observability and describe how requests move through a system, capturing each step's timing and status.", + "group": "primary", + "requirement": "recommended", + "object_type": "trace", + "object_name": "Trace", + "caption": "Trace" + } + }, + "meta": "profile", + "name": "trace", + "description": "The Trace Profile extends the OCSF framework to capture and standardize observability events, specifically targeting trace-level data. This profile enables integration and normalization of distributed tracing information, allowing OCSF events to retain essential trace context such as trace IDs, span relationships, and service dependencies.", + "caption": "Trace", + "annotations": { + "group": "primary" + } +} diff --git a/crates/openshell-ocsf/src/builders/api_activity.rs b/crates/openshell-ocsf/src/builders/api_activity.rs index 4740b05053..941fb46a43 100644 --- a/crates/openshell-ocsf/src/builders/api_activity.rs +++ b/crates/openshell-ocsf/src/builders/api_activity.rs @@ -29,6 +29,7 @@ fn sanitize_model_name(name: &str) -> String { /// Builder for API Activity [6003] events. pub struct ApiActivityBuilder<'a> { ctx: ContextRef<'a>, + trace: Option, severity: SeverityId, status: Option, message: Option, @@ -48,10 +49,18 @@ pub struct ApiActivityBuilder<'a> { } impl<'a> ApiActivityBuilder<'a> { + /// Set trace correlation explicitly instead of using the active span. + #[must_use] + pub fn trace(mut self, trace: crate::TraceCorrelation) -> Self { + self.trace = Some(trace); + self + } + #[must_use] pub fn new(ctx: impl Into>, operation: impl Into) -> Self { Self { ctx: ctx.into(), + trace: None, severity: SeverityId::Informational, status: None, message: None, @@ -201,6 +210,10 @@ impl<'a> ApiActivityBuilder<'a> { self.ctx .apply_common_fields(&mut base, self.status, self.message); + if let Some(trace) = self.trace { + base.set_trace(trace); + } + OcsfEvent::ApiActivity(ApiActivityEvent { base, api: Api { diff --git a/crates/openshell-ocsf/src/builders/authentication.rs b/crates/openshell-ocsf/src/builders/authentication.rs index 492bdf65b7..6b997e6455 100644 --- a/crates/openshell-ocsf/src/builders/authentication.rs +++ b/crates/openshell-ocsf/src/builders/authentication.rs @@ -12,6 +12,7 @@ use crate::objects::{Endpoint, HttpRequest, Service, User}; /// Builder for Authentication [3002] Logon events. pub struct AuthenticationBuilder<'a> { ctx: ContextRef<'a>, + trace: Option, user: User, service: Service, auth_protocol: Option, @@ -28,10 +29,18 @@ pub struct AuthenticationBuilder<'a> { } impl<'a> AuthenticationBuilder<'a> { + /// Set trace correlation explicitly instead of using the active span. + #[must_use] + pub fn trace(mut self, trace: crate::TraceCorrelation) -> Self { + self.trace = Some(trace); + self + } + #[must_use] pub fn new(ctx: impl Into>, user: User, service: Service) -> Self { Self { ctx: ctx.into(), + trace: None, user, service, auth_protocol: None, @@ -116,6 +125,9 @@ impl<'a> AuthenticationBuilder<'a> { // Not `apply_common_fields`: the container profile defines `container` // on endpoints and processes, never at the event top level. base.set_device(self.ctx.device()); + if let Some(trace) = self.trace { + base.set_trace(trace); + } OcsfEvent::Authentication(AuthenticationEvent { base, diff --git a/crates/openshell-ocsf/src/builders/base.rs b/crates/openshell-ocsf/src/builders/base.rs index c08f1aa0aa..5243caac7a 100644 --- a/crates/openshell-ocsf/src/builders/base.rs +++ b/crates/openshell-ocsf/src/builders/base.rs @@ -11,6 +11,7 @@ use crate::events::{BaseEvent, OcsfEvent}; /// Builder for Base Event [0] — events without a specific OCSF class. pub struct BaseEventBuilder<'a> { ctx: ContextRef<'a>, + trace: Option, severity: SeverityId, status: Option, message: Option, @@ -19,10 +20,18 @@ pub struct BaseEventBuilder<'a> { } impl<'a> BaseEventBuilder<'a> { + /// Set trace correlation explicitly instead of using the active span. + #[must_use] + pub fn trace(mut self, trace: crate::TraceCorrelation) -> Self { + self.trace = Some(trace); + self + } + #[must_use] pub fn new(ctx: impl Into>) -> Self { Self { ctx: ctx.into(), + trace: None, severity: SeverityId::Informational, status: None, message: None, @@ -63,6 +72,10 @@ impl<'a> BaseEventBuilder<'a> { self.ctx .apply_common_fields(&mut base, self.status, self.message); + if let Some(trace) = self.trace { + base.set_trace(trace); + } + OcsfEvent::Base(BaseEvent { base }) } } diff --git a/crates/openshell-ocsf/src/builders/config.rs b/crates/openshell-ocsf/src/builders/config.rs index 906470408d..25e229d04e 100644 --- a/crates/openshell-ocsf/src/builders/config.rs +++ b/crates/openshell-ocsf/src/builders/config.rs @@ -11,6 +11,7 @@ use crate::events::{DeviceConfigStateChangeEvent, OcsfEvent}; /// Builder for Device Config State Change [5019] events. pub struct ConfigStateChangeBuilder<'a> { ctx: ContextRef<'a>, + trace: Option, severity: SeverityId, status: Option, state_id: Option, @@ -22,10 +23,18 @@ pub struct ConfigStateChangeBuilder<'a> { } impl<'a> ConfigStateChangeBuilder<'a> { + /// Set trace correlation explicitly instead of using the active span. + #[must_use] + pub fn trace(mut self, trace: crate::TraceCorrelation) -> Self { + self.trace = Some(trace); + self + } + #[must_use] pub fn new(ctx: impl Into>) -> Self { Self { ctx: ctx.into(), + trace: None, severity: SeverityId::Informational, status: None, state_id: None, @@ -82,6 +91,10 @@ impl<'a> ConfigStateChangeBuilder<'a> { self.ctx .apply_common_fields(&mut base, self.status, self.message); + if let Some(trace) = self.trace { + base.set_trace(trace); + } + OcsfEvent::DeviceConfigStateChange(DeviceConfigStateChangeEvent { base, state: self.state_id, diff --git a/crates/openshell-ocsf/src/builders/finding.rs b/crates/openshell-ocsf/src/builders/finding.rs index ccb45d29a8..ba8e01d285 100644 --- a/crates/openshell-ocsf/src/builders/finding.rs +++ b/crates/openshell-ocsf/src/builders/finding.rs @@ -12,6 +12,7 @@ use crate::objects::{Attack, Evidence, FindingInfo, Remediation}; /// Builder for Detection Finding [2004] events. pub struct DetectionFindingBuilder<'a> { ctx: ContextRef<'a>, + trace: Option, activity: ActivityId, severity: SeverityId, action: Option, @@ -30,10 +31,18 @@ pub struct DetectionFindingBuilder<'a> { } impl<'a> DetectionFindingBuilder<'a> { + /// Set trace correlation explicitly instead of using the active span. + #[must_use] + pub fn trace(mut self, trace: crate::TraceCorrelation) -> Self { + self.trace = Some(trace); + self + } + #[must_use] pub fn new(ctx: impl Into>) -> Self { Self { ctx: ctx.into(), + trace: None, activity: ActivityId::Open, severity: SeverityId::Medium, action: None, @@ -151,6 +160,10 @@ impl<'a> DetectionFindingBuilder<'a> { } self.ctx.apply_common_fields(&mut base, None, self.message); + if let Some(trace) = self.trace { + base.set_trace(trace); + } + OcsfEvent::DetectionFinding(DetectionFindingEvent { base, finding_info: self diff --git a/crates/openshell-ocsf/src/builders/http.rs b/crates/openshell-ocsf/src/builders/http.rs index 86a22f63ac..b0c8bef1b0 100644 --- a/crates/openshell-ocsf/src/builders/http.rs +++ b/crates/openshell-ocsf/src/builders/http.rs @@ -12,6 +12,7 @@ use crate::objects::{Actor, Endpoint, FirewallRule, HttpRequest, HttpResponse}; /// Builder for HTTP Activity [4002] events. pub struct HttpActivityBuilder<'a, Context = MissingHttpContext> { ctx: ContextRef<'a>, + trace: Option, activity: ActivityId, action: Option, disposition: Option, @@ -59,6 +60,7 @@ impl<'a> HttpActivityBuilder<'a, MissingHttpContext> { pub fn new(ctx: impl Into>) -> Self { Self { ctx: ctx.into(), + trace: None, activity: ActivityId::Unknown, action: None, disposition: None, @@ -103,9 +105,17 @@ impl HttpActivityBuilder<'_, HasHttpContext> { } impl<'a, Context> HttpActivityBuilder<'a, Context> { + /// Set trace correlation explicitly instead of using the active span. + #[must_use] + pub fn trace(mut self, trace: crate::TraceCorrelation) -> Self { + self.trace = Some(trace); + self + } + fn with_http_request(self, req: HttpRequest) -> HttpActivityBuilder<'a, HasHttpContext> { HttpActivityBuilder { ctx: self.ctx, + trace: self.trace, activity: self.activity, action: self.action, disposition: self.disposition, @@ -127,6 +137,7 @@ impl<'a, Context> HttpActivityBuilder<'a, Context> { fn with_http_response(self, resp: HttpResponse) -> HttpActivityBuilder<'a, HasHttpContext> { HttpActivityBuilder { ctx: self.ctx, + trace: self.trace, activity: self.activity, action: self.action, disposition: self.disposition, @@ -242,6 +253,10 @@ impl HttpActivityBuilder<'_, HasHttpContext> { self.ctx .apply_common_fields(&mut base, self.status, self.message); + if let Some(trace) = self.trace { + base.set_trace(trace); + } + OcsfEvent::HttpActivity(HttpActivityEvent { base, http_request: self.http_request, diff --git a/crates/openshell-ocsf/src/builders/lifecycle.rs b/crates/openshell-ocsf/src/builders/lifecycle.rs index b425b3e79b..f0a2357179 100644 --- a/crates/openshell-ocsf/src/builders/lifecycle.rs +++ b/crates/openshell-ocsf/src/builders/lifecycle.rs @@ -12,6 +12,7 @@ use crate::objects::Product; /// Builder for Application Lifecycle [6002] events. pub struct AppLifecycleBuilder<'a> { ctx: ContextRef<'a>, + trace: Option, activity: ActivityId, severity: SeverityId, status: Option, @@ -19,10 +20,18 @@ pub struct AppLifecycleBuilder<'a> { } impl<'a> AppLifecycleBuilder<'a> { + /// Set trace correlation explicitly instead of using the active span. + #[must_use] + pub fn trace(mut self, trace: crate::TraceCorrelation) -> Self { + self.trace = Some(trace); + self + } + #[must_use] pub fn new(ctx: impl Into>) -> Self { Self { ctx: ctx.into(), + trace: None, activity: ActivityId::Unknown, severity: SeverityId::Informational, status: None, @@ -46,6 +55,10 @@ impl<'a> AppLifecycleBuilder<'a> { self.ctx .apply_common_fields(&mut base, self.status, self.message); + if let Some(trace) = self.trace { + base.set_trace(trace); + } + OcsfEvent::ApplicationLifecycle(ApplicationLifecycleEvent { base, app: Product::openshell_sandbox(&self.ctx.product_version), diff --git a/crates/openshell-ocsf/src/builders/network.rs b/crates/openshell-ocsf/src/builders/network.rs index 7907abe4a6..bfdcbaf7cd 100644 --- a/crates/openshell-ocsf/src/builders/network.rs +++ b/crates/openshell-ocsf/src/builders/network.rs @@ -12,6 +12,7 @@ use crate::objects::{Actor, ConnectionInfo, Endpoint, FirewallRule}; /// Builder for Network Activity [4001] events. pub struct NetworkActivityBuilder<'a, EndpointState = MissingNetworkEndpoint> { ctx: ContextRef<'a>, + trace: Option, activity: ActivityId, activity_name: Option, action: Option, @@ -61,6 +62,7 @@ impl<'a> NetworkActivityBuilder<'a, MissingNetworkEndpoint> { pub fn new(ctx: impl Into>) -> Self { Self { ctx: ctx.into(), + trace: None, activity: ActivityId::Unknown, activity_name: None, action: None, @@ -114,12 +116,20 @@ impl NetworkActivityBuilder<'_, HasNetworkEndpoint> { } impl<'a, EndpointState> NetworkActivityBuilder<'a, EndpointState> { + /// Set trace correlation explicitly instead of using the active span. + #[must_use] + pub fn trace(mut self, trace: crate::TraceCorrelation) -> Self { + self.trace = Some(trace); + self + } + fn with_src_endpoint( self, endpoint: Endpoint, ) -> NetworkActivityBuilder<'a, HasNetworkEndpoint> { NetworkActivityBuilder { ctx: self.ctx, + trace: self.trace, activity: self.activity, activity_name: self.activity_name, action: self.action, @@ -146,6 +156,7 @@ impl<'a, EndpointState> NetworkActivityBuilder<'a, EndpointState> { ) -> NetworkActivityBuilder<'a, HasNetworkEndpoint> { NetworkActivityBuilder { ctx: self.ctx, + trace: self.trace, activity: self.activity, activity_name: self.activity_name, action: self.action, @@ -284,6 +295,10 @@ impl NetworkActivityBuilder<'_, HasNetworkEndpoint> { self.ctx .apply_common_fields(&mut base, self.status, self.message); + if let Some(trace) = self.trace { + base.set_trace(trace); + } + OcsfEvent::NetworkActivity(NetworkActivityEvent { base, src_endpoint: self.src_endpoint, diff --git a/crates/openshell-ocsf/src/builders/process.rs b/crates/openshell-ocsf/src/builders/process.rs index 5acdb75f01..1e65e189a0 100644 --- a/crates/openshell-ocsf/src/builders/process.rs +++ b/crates/openshell-ocsf/src/builders/process.rs @@ -12,6 +12,7 @@ use crate::objects::{Actor, Process}; /// Builder for Process Activity [1007] events. pub struct ProcessActivityBuilder<'a> { ctx: ContextRef<'a>, + trace: Option, activity: ActivityId, severity: SeverityId, status: Option, @@ -25,10 +26,18 @@ pub struct ProcessActivityBuilder<'a> { } impl<'a> ProcessActivityBuilder<'a> { + /// Set trace correlation explicitly instead of using the active span. + #[must_use] + pub fn trace(mut self, trace: crate::TraceCorrelation) -> Self { + self.trace = Some(trace); + self + } + #[must_use] pub fn new(ctx: impl Into>) -> Self { Self { ctx: ctx.into(), + trace: None, activity: ActivityId::Unknown, severity: SeverityId::Informational, status: None, @@ -75,6 +84,10 @@ impl<'a> ProcessActivityBuilder<'a> { self.ctx .apply_common_fields(&mut base, self.status, self.message); + if let Some(trace) = self.trace { + base.set_trace(trace); + } + OcsfEvent::ProcessActivity(ProcessActivityEvent { base, process: self.process.unwrap_or_else(|| Process::new("unknown", 0)), diff --git a/crates/openshell-ocsf/src/builders/ssh.rs b/crates/openshell-ocsf/src/builders/ssh.rs index 2777bbd05b..b7356a3f03 100644 --- a/crates/openshell-ocsf/src/builders/ssh.rs +++ b/crates/openshell-ocsf/src/builders/ssh.rs @@ -12,6 +12,7 @@ use crate::objects::{Actor, Endpoint}; /// Builder for SSH Activity [4007] events. pub struct SshActivityBuilder<'a> { ctx: ContextRef<'a>, + trace: Option, activity: ActivityId, action: Option, disposition: Option, @@ -27,10 +28,18 @@ pub struct SshActivityBuilder<'a> { } impl<'a> SshActivityBuilder<'a> { + /// Set trace correlation explicitly instead of using the active span. + #[must_use] + pub fn trace(mut self, trace: crate::TraceCorrelation) -> Self { + self.trace = Some(trace); + self + } + #[must_use] pub fn new(ctx: impl Into>) -> Self { Self { ctx: ctx.into(), + trace: None, activity: ActivityId::Unknown, action: None, disposition: None, @@ -77,6 +86,10 @@ impl<'a> SshActivityBuilder<'a> { self.ctx .apply_common_fields(&mut base, self.status, self.message); + if let Some(trace) = self.trace { + base.set_trace(trace); + } + OcsfEvent::SshActivity(SshActivityEvent { base, src_endpoint: self.src_endpoint, diff --git a/crates/openshell-ocsf/src/events/base_event.rs b/crates/openshell-ocsf/src/events/base_event.rs index 78bd927612..6ada361020 100644 --- a/crates/openshell-ocsf/src/events/base_event.rs +++ b/crates/openshell-ocsf/src/events/base_event.rs @@ -13,6 +13,10 @@ use crate::objects::{AiModel, Container, Device, Metadata}; /// Every event class embeds this struct via `#[serde(flatten)]`. #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] pub struct BaseEventData { + /// Optional trace correlation from the OCSF producer (Trace profile). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trace: Option, + /// OCSF class UID (e.g., 4001 for Network Activity). pub class_uid: u32, @@ -80,7 +84,6 @@ impl Serialize for BaseEventData { fn serialize(&self, serializer: S) -> Result { use serde::ser::SerializeMap; - // Count fields: 9 required + severity pair (2) + up to 6 optional let mut map = serializer.serialize_map(None)?; map.serialize_entry("class_uid", &self.class_uid)?; @@ -119,6 +122,9 @@ impl Serialize for BaseEventData { if let Some(ref ai_model) = self.ai_model { map.serialize_entry("ai_model", ai_model)?; } + if let Some(ref trace) = self.trace { + map.serialize_entry("trace", trace)?; + } if let Some(ref unmapped) = self.unmapped { map.serialize_entry("unmapped", unmapped)?; } @@ -162,6 +168,7 @@ impl BaseEventData { device: None, container: None, ai_model: None, + trace: None, unmapped: None, } } @@ -201,6 +208,19 @@ impl BaseEventData { self.ai_model = Some(ai_model); } + /// Set explicit trace correlation and add the Trace profile once. + pub fn set_trace(&mut self, trace: crate::TraceCorrelation) { + self.trace = Some(trace); + if !self + .metadata + .profiles + .iter() + .any(|profile| profile == "trace") + { + self.metadata.profiles.push("trace".to_string()); + } + } + /// Add an unmapped field. pub fn add_unmapped(&mut self, key: &str, value: impl Into) { let map = self diff --git a/crates/openshell-ocsf/src/events/mod.rs b/crates/openshell-ocsf/src/events/mod.rs index 7dca70e953..79f80e3d6c 100644 --- a/crates/openshell-ocsf/src/events/mod.rs +++ b/crates/openshell-ocsf/src/events/mod.rs @@ -156,6 +156,22 @@ impl OcsfEvent { Self::Base(e) => &e.base, } } + + /// Returns mutable base data common to all event classes. + pub(crate) fn base_mut(&mut self) -> &mut BaseEventData { + match self { + Self::NetworkActivity(e) => &mut e.base, + Self::HttpActivity(e) => &mut e.base, + Self::SshActivity(e) => &mut e.base, + Self::ProcessActivity(e) => &mut e.base, + Self::Authentication(e) => &mut e.base, + Self::DetectionFinding(e) => &mut e.base, + Self::ApplicationLifecycle(e) => &mut e.base, + Self::DeviceConfigStateChange(e) => &mut e.base, + Self::ApiActivity(e) => &mut e.base, + Self::Base(e) => &mut e.base, + } + } } #[cfg(test)] diff --git a/crates/openshell-ocsf/src/format/downgrade.rs b/crates/openshell-ocsf/src/format/downgrade.rs index 06bd4edd24..bad92c6145 100644 --- a/crates/openshell-ocsf/src/format/downgrade.rs +++ b/crates/openshell-ocsf/src/format/downgrade.rs @@ -42,10 +42,10 @@ impl std::str::FromStr for OcsfSchemaVersion { } /// Fields to strip when downgrading to v1.3.0 or earlier. -const STRIP_FOR_V1_3: &[&str] = &["ai_model", "container", "observation_point_id"]; +const STRIP_FOR_V1_3: &[&str] = &["ai_model", "container", "observation_point_id", "trace"]; /// Profile names to remove from `metadata.profiles` when downgrading to v1.3.0 or earlier. -const STRIP_PROFILES_V1_3: &[&str] = &["ai_operation", "container"]; +const STRIP_PROFILES_V1_3: &[&str] = &["ai_operation", "container", "trace"]; /// Downgrade a serialized OCSF event to the target schema version. /// @@ -155,6 +155,41 @@ mod tests { }) } + fn assert_trace_removed(target: &str) { + let ctx = crate::builders::test_sandbox_context(); + let mut event = crate::NetworkActivityBuilder::new(&ctx) + .action(crate::ActionId::Denied) + .src_endpoint_addr("192.0.2.5".parse().unwrap(), 51234) + .dst_endpoint(crate::Endpoint::from_ip("192.0.2.1".parse().unwrap(), 443)) + .trace(crate::TraceCorrelation { + uid: "0af7651916cd43dd8448eb211c80319c".to_string(), + }) + .build() + .to_json() + .unwrap(); + assert!(event.get("trace").is_some()); + assert!(downgrade_event(&mut event, target)); + assert!(event.get("trace").is_none()); + assert!( + !event["metadata"]["profiles"] + .as_array() + .unwrap() + .iter() + .any(|name| name == "trace") + ); + assert_eq!(event["metadata"]["version"], target); + } + + #[test] + fn trace_object_and_profile_are_removed_for_1_3() { + assert_trace_removed("1.3.0"); + } + + #[test] + fn trace_object_and_profile_are_removed_for_1_1() { + assert_trace_removed("1.1.0"); + } + #[test] fn test_downgrade_to_v1_3_strips_fields() { let mut event = test_event(); diff --git a/crates/openshell-ocsf/src/lib.rs b/crates/openshell-ocsf/src/lib.rs index 310cd9afe4..29cc75b9eb 100644 --- a/crates/openshell-ocsf/src/lib.rs +++ b/crates/openshell-ocsf/src/lib.rs @@ -31,6 +31,9 @@ pub mod events; pub mod format; pub mod objects; pub mod sanitize; +mod trace; + +pub use trace::{TraceCorrelation, register_trace_correlation_extractor}; pub mod tracing_layers; #[cfg(any(test, feature = "test-support"))] diff --git a/crates/openshell-ocsf/src/trace.rs b/crates/openshell-ocsf/src/trace.rs new file mode 100644 index 0000000000..30b911764d --- /dev/null +++ b/crates/openshell-ocsf/src/trace.rs @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! OCSF trace correlation and optional enrichment at emission. + +use std::sync::OnceLock; + +use serde::{Deserialize, Serialize}; + +/// Trace ID supplied by an OCSF producer. +/// +/// The registered extractor returns valid, sampled context from the active +/// span. Gateway context may inherit a caller's traceparent, including before +/// authentication: this value is a correlation hint, not an identity assertion. +/// Explicit callers are responsible for validating their trace IDs. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TraceCorrelation { + /// W3C trace ID: 32 lowercase hexadecimal characters. + pub uid: String, +} + +static EXTRACTOR: OnceLock Option> = OnceLock::new(); + +/// Register the process-wide trace extractor at startup. +/// +/// Returns `false` if an extractor is already registered. Without an extractor, +/// events receive only explicitly supplied correlation. +pub fn register_trace_correlation_extractor(extractor: fn() -> Option) -> bool { + EXTRACTOR.set(extractor).is_ok() +} + +pub fn enrich(event: &mut crate::OcsfEvent) { + if event.base().trace.is_none() + && let Some(trace) = EXTRACTOR.get().and_then(|extract| extract()) + { + event.base_mut().set_trace(trace); + } +} + +#[cfg(test)] +mod tests { + use crate::validation::schema::{ + load_class_schema, load_object_schema, validate_known_fields, validate_required_fields, + }; + use crate::*; + + #[test] + fn all_builders_preserve_explicit_trace_through_typestate_and_roundtrip() { + let ctx = builders::test_sandbox_context(); + let trace = TraceCorrelation { + uid: "0af7651916cd43dd8448eb211c80319c".to_string(), + }; + for context in [ + ContextRef::from(&ctx), + ContextRef::from(std::sync::Arc::new(ctx.clone())), + ] { + let events = [ + NetworkActivityBuilder::new(context.clone()) + .trace(trace.clone()) + .dst_endpoint(Endpoint::from_domain("example.com", 443)) + .build(), + HttpActivityBuilder::new(context.clone()) + .trace(trace.clone()) + .http_request(HttpRequest::new( + "GET", + Url::new("https", "example.com", "/", 443), + )) + .build(), + BaseEventBuilder::new(context.clone()) + .trace(trace.clone()) + .build(), + AppLifecycleBuilder::new(context.clone()) + .trace(trace.clone()) + .build(), + ConfigStateChangeBuilder::new(context.clone()) + .trace(trace.clone()) + .build(), + DetectionFindingBuilder::new(context.clone()) + .trace(trace.clone()) + .build(), + ProcessActivityBuilder::new(context.clone()) + .trace(trace.clone()) + .actor_process(Process::new("openshell-sandbox", 1)) + .process(Process::new("workload", 2)) + .build(), + SshActivityBuilder::new(context.clone()) + .trace(trace.clone()) + .dst_endpoint(Endpoint::from_ip("192.0.2.1".parse().unwrap(), 22)) + .build(), + AuthenticationBuilder::new( + context.clone(), + User::unidentified(), + Service::new("openshell-gateway"), + ) + .trace(trace.clone()) + .build(), + ApiActivityBuilder::new(context, "GET /") + .trace(trace.clone()) + .build(), + ]; + let schema = load_object_schema("trace"); + let profile: serde_json::Value = + serde_json::from_str(include_str!("../schemas/ocsf/v1.8.0/profiles/trace.json")) + .unwrap(); + assert_eq!(profile["attributes"]["trace"]["object_type"], "trace"); + for event in events { + let value = event.to_json().unwrap(); + assert_eq!(value["device"]["hostname"], ctx.hostname); + if value["class_uid"] == 3002 { + // Authentication's schema does not allow the container profile. + assert!(value.get("container").is_none()); + } else { + assert_eq!(value["container"]["uid"], ctx.sandbox_id); + assert_eq!(value["container"]["name"], ctx.sandbox_name); + } + let class = match value["class_uid"].as_u64().unwrap() { + 0 => "base_event", + 4001 => "network_activity", + 4002 => "http_activity", + 4007 => "ssh_activity", + 1007 => "process_activity", + 3002 => "authentication", + 2004 => "detection_finding", + 6002 => "application_lifecycle", + 5019 => "device_config_state_change", + 6003 => "api_activity", + uid => panic!("missing schema coverage for class {uid}"), + }; + let class_schema = load_class_schema(class); + validate_required_fields(&value, &class_schema); + // Check correlation separately from native conformance work in #4283. + validate_known_fields( + &serde_json::json!({ + "metadata": value["metadata"], + "trace": value["trace"], + }), + &class_schema, + ); + validate_required_fields(&value["trace"], &schema); + assert_eq!(value["trace"]["uid"], trace.uid); + assert_eq!(value["trace"].as_object().unwrap().len(), 1); + let decoded: OcsfEvent = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(decoded.to_json().unwrap(), value); + assert_eq!( + value["metadata"]["profiles"] + .as_array() + .unwrap() + .iter() + .filter(|p| *p == "trace") + .count(), + 1 + ); + let mut plain = value; + plain.as_object_mut().unwrap().remove("trace"); + plain["metadata"]["profiles"] + .as_array_mut() + .unwrap() + .retain(|name| name != "trace"); + validate_required_fields(&plain, &class_schema); + } + } + } + + #[test] + fn every_event_class_vendors_the_optional_trace_profile_attribute() { + let profile: serde_json::Value = + serde_json::from_str(include_str!("../schemas/ocsf/v1.8.0/profiles/trace.json")) + .unwrap(); + let mut attribute = profile["attributes"]["trace"].clone(); + attribute["profiles"] = serde_json::json!(["trace"]); + for class in [ + "base_event", + "network_activity", + "http_activity", + "ssh_activity", + "process_activity", + "detection_finding", + "application_lifecycle", + "device_config_state_change", + "api_activity", + "authentication", + ] { + let schema = load_class_schema(class); + assert_eq!(schema["attributes"]["trace"], attribute, "{class}"); + assert_ne!(schema["attributes"]["trace"]["requirement"], "required"); + assert!( + schema["profiles"] + .as_array() + .unwrap() + .iter() + .any(|name| name == "trace"), + "{class} must support the Trace profile" + ); + } + } + + #[test] + fn trace_profile_object_references_are_vendored() { + let mut pending = vec!["trace".to_string()]; + let mut seen = std::collections::BTreeSet::new(); + while let Some(name) = pending.pop() { + if !seen.insert(name.clone()) { + continue; + } + let schema = load_object_schema(&name); + for attribute in schema["attributes"].as_object().unwrap().values() { + if let Some(object) = attribute["object_type"].as_str() { + pending.push(object.to_string()); + } + } + } + assert!(seen.contains("span")); + let span = load_object_schema("span"); + for field in ["start_time", "end_time"] { + assert_eq!(span["attributes"][field]["requirement"], "required"); + } + } + + #[test] + fn plain_event_omits_trace_profile_and_object() { + let ctx = builders::test_sandbox_context(); + let value = BaseEventBuilder::new(&ctx).build().to_json().unwrap(); + assert!(value.get("trace").is_none()); + assert!( + !value["metadata"]["profiles"] + .as_array() + .unwrap() + .iter() + .any(|p| p == "trace") + ); + } +} diff --git a/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs b/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs index c4a5ccb895..ca5a5e9cdd 100644 --- a/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs +++ b/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs @@ -34,7 +34,8 @@ pub fn clone_current_event() -> Option { /// as human-readable shorthand and structured JSONL for their configured sinks. /// /// Both layers receive the event — `clone_current_event()` is non-consuming. -pub fn emit_ocsf_event(event: OcsfEvent) { +pub fn emit_ocsf_event(mut event: OcsfEvent) { + crate::trace::enrich(&mut event); // Store the event in thread-local so layers can access it set_current_event(event); @@ -81,7 +82,8 @@ pub fn clear_current_event() { /// bare `emit_ocsf_event`, but the gateway hosts many sandboxes, so it stamps a /// per-event `sandbox_id` field here instead. One tracing event feeds both the /// JSONL audit file and the routing bus. -pub fn emit_ocsf_event_routed(sandbox_id: &str, event: OcsfEvent) { +pub fn emit_ocsf_event_routed(sandbox_id: &str, mut event: OcsfEvent) { + crate::trace::enrich(&mut event); let message = event.format_shorthand(); set_current_event(event); tracing::info!(target: "ocsf", sandbox_id = %sandbox_id, message = %message); diff --git a/crates/openshell-otel/src/correlation.rs b/crates/openshell-otel/src/correlation.rs new file mode 100644 index 0000000000..7c01c067c8 --- /dev/null +++ b/crates/openshell-otel/src/correlation.rs @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sampled trace correlation for OCSF producers. + +use opentelemetry::trace::TraceContextExt as _; +use tracing_opentelemetry::OpenTelemetrySpanExt as _; + +/// Extract the active span's W3C trace ID for an OCSF event. +/// +/// Returns `None` unless the current span has valid, sampled context. This reads +/// the current span without extracting workload HTTP headers or environment +/// variables. The active span may already inherit a gateway caller's context, +/// including before authentication. This is a correlation hint, not proof of +/// origin. A sampled ID does not guarantee export or retention. +#[must_use] +pub fn current_ocsf_trace_correlation() -> Option { + let context = tracing::Span::current().context(); + let span = context.span(); + let span_context = span.span_context(); + (span_context.is_valid() && span_context.is_sampled()).then(|| { + openshell_ocsf::TraceCorrelation { + uid: span_context.trace_id().to_string(), + } + }) +} diff --git a/crates/openshell-otel/src/lib.rs b/crates/openshell-otel/src/lib.rs index 914bbea861..b89c5f2805 100644 --- a/crates/openshell-otel/src/lib.rs +++ b/crates/openshell-otel/src/lib.rs @@ -3,6 +3,9 @@ //! Shared OpenTelemetry trace and log export support for `OpenShell` services. +mod correlation; +pub use correlation::current_ocsf_trace_correlation; + mod driver; mod grpc; mod logs; @@ -40,6 +43,18 @@ use tracing_subscriber::Layer as _; use tracing_subscriber::layer::{Context, Filter}; use tracing_subscriber::registry::LookupSpan; +/// Export supervisor spans at INFO by default, or DEBUG and TRACE when enabled. +/// Dependency spans remain at INFO. +#[must_use] +pub fn supervisor_span_filter(log_level: &str) -> tracing_subscriber::EnvFilter { + use tracing_subscriber::filter::LevelFilter; + let level = log_level + .parse::() + .unwrap_or(LevelFilter::INFO) + .max(LevelFilter::INFO); + tracing_subscriber::EnvFilter::new(format!("info,openshell={level}")) +} + const SDK_UNKNOWN_SERVICE_PREFIX: &str = "unknown_service"; /// Build the resource attributes shared by gateway and compute-driver providers. diff --git a/crates/openshell-otel/src/ocsf.rs b/crates/openshell-otel/src/ocsf.rs index 2dc33cee8f..2c1d85ce45 100644 --- a/crates/openshell-otel/src/ocsf.rs +++ b/crates/openshell-otel/src/ocsf.rs @@ -509,6 +509,42 @@ mod tests { assert!(body.get("container").is_none()); } + #[test] + fn trace_body_obeys_native_and_older_schema_contracts() { + let event = ConfigStateChangeBuilder::new(&event_context("sandbox-1")) + .trace(openshell_ocsf::TraceCorrelation { + uid: "0af7651916cd43dd8448eb211c80319c".to_string(), + }) + .build(); + let native_body = body(&record(&event, None)); + assert_eq!(native_body, native(&event)); + assert_eq!( + native_body["trace"]["uid"], + "0af7651916cd43dd8448eb211c80319c" + ); + assert!( + native_body["metadata"]["profiles"] + .as_array() + .unwrap() + .iter() + .any(|profile| profile == "trace") + ); + + for target in [OcsfSchemaVersion::V1_1, OcsfSchemaVersion::V1_3] { + let downgraded = body(&record(&event, Some(target))); + assert_eq!(downgraded["metadata"]["version"], target.as_str()); + assert!(downgraded.get("trace").is_none()); + assert!( + !downgraded["metadata"]["profiles"] + .as_array() + .unwrap() + .iter() + .any(|profile| profile == "trace") + ); + } + assert_eq!(native(&event), native_body); + } + #[test] fn timestamps_are_event_time_and_observed_time() { let event = event("sandbox-1", SeverityId::Informational); diff --git a/crates/openshell-otel/tests/ocsf_correlation.rs b/crates/openshell-otel/tests/ocsf_correlation.rs new file mode 100644 index 0000000000..55874c7f80 --- /dev/null +++ b/crates/openshell-otel/tests/ocsf_correlation.rs @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::{Arc, LazyLock, Mutex}; + +use openshell_ocsf::{BaseEventBuilder, OcsfJsonlLayer, TraceCorrelation, ocsf_emit}; +use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, Sampler, SdkTracerProvider}; +use tracing_subscriber::prelude::*; + +static SETUP: LazyLock<()> = LazyLock::new(|| { + tracing::subscriber::set_global_default(tracing_subscriber::registry()).unwrap(); + assert!(openshell_ocsf::register_trace_correlation_extractor( + openshell_otel::current_ocsf_trace_correlation, + )); +}); + +#[derive(Clone)] +struct Writer(Arc>>); +impl std::io::Write for Writer { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +fn capture( + sampler: Sampler, + active: bool, + explicit: Option, +) -> (serde_json::Value, Vec) { + capture_route(sampler, active, explicit, false) +} + +fn capture_route( + sampler: Sampler, + active: bool, + explicit: Option, + routed: bool, +) -> (serde_json::Value, Vec) { + capture_context(sampler, active, explicit, routed, None) +} + +fn capture_context( + sampler: Sampler, + active: bool, + explicit: Option, + routed: bool, + remote_parent: Option, +) -> (serde_json::Value, Vec) { + LazyLock::force(&SETUP); + let exporter = InMemorySpanExporterBuilder::new().build(); + let provider = SdkTracerProvider::builder() + .with_sampler(sampler) + .with_simple_exporter(exporter.clone()) + .build(); + let bytes = Arc::new(Mutex::new(vec![])); + let subscriber = tracing_subscriber::registry() + .with(openshell_otel::layer(&provider, "ocsf-test")) + .with(OcsfJsonlLayer::new(Writer(bytes.clone()))); + tracing::subscriber::with_default(subscriber, || { + let gateway_request = remote_parent.is_some(); + let emit = || { + let ctx = openshell_ocsf::ctx::ctx(); + if gateway_request { + ocsf_emit!( + openshell_ocsf::AuthenticationBuilder::new( + ctx, + openshell_ocsf::User::unidentified(), + openshell_ocsf::Service::new("openshell-gateway"), + ) + .status(openshell_ocsf::StatusId::Failure) + .build() + ); + return; + } + let mut builder = BaseEventBuilder::new(ctx); + if let Some(trace) = explicit { + builder = builder.trace(trace); + } + if routed { + openshell_ocsf::emit_ocsf_event_routed("test-sandbox", builder.build()); + } else { + ocsf_emit!(builder.build()); + } + }; + if active { + let span = tracing::info_span!("sampled-operation"); + if let Some(parent) = remote_parent { + use opentelemetry::trace::TraceContextExt as _; + use tracing_opentelemetry::OpenTelemetrySpanExt as _; + span.set_parent(opentelemetry::Context::new().with_remote_span_context(parent)) + .unwrap(); + } + span.in_scope(emit); + } else { + emit(); + } + }); + let value = serde_json::from_slice(&bytes.lock().unwrap()).unwrap(); + provider.force_flush().unwrap(); + let spans = exporter.get_finished_spans().unwrap(); + provider.shutdown().unwrap(); + (value, spans) +} + +#[test] +fn authentication_failure_correlation_can_inherit_sampled_or_unsampled_caller_context() { + use opentelemetry::trace::{SpanContext, SpanId, TraceFlags, TraceId, TraceState}; + let uid = "0af7651916cd43dd8448eb211c80319c"; + for sampled in [true, false] { + let parent = SpanContext::new( + TraceId::from_hex(uid).unwrap(), + SpanId::from_hex("b7ad6b7169203331").unwrap(), + if sampled { + TraceFlags::SAMPLED + } else { + TraceFlags::default() + }, + true, + TraceState::default(), + ); + let (event, spans) = capture_context( + Sampler::ParentBased(Box::new(Sampler::AlwaysOn)), + true, + None, + false, + Some(parent), + ); + assert_eq!(event["class_uid"], 3002); + assert_eq!(event["status_id"], 2); + if sampled { + assert_eq!(event["trace"]["uid"], uid); + assert_eq!(spans.len(), 1); + assert_eq!(spans[0].span_context.trace_id().to_string(), uid); + } else { + assert_no_trace(&event); + assert!(spans.is_empty()); + } + } +} + +fn assert_no_trace(value: &serde_json::Value) { + assert!(value.get("trace").is_none()); + assert!( + !value["metadata"]["profiles"] + .as_array() + .unwrap() + .iter() + .any(|p| p == "trace") + ); +} + +#[test] +fn sampled_active_span_correlates_to_exported_w3c_trace() { + let (value, spans) = capture(Sampler::AlwaysOn, true, None); + let uid = value["trace"]["uid"].as_str().unwrap(); + assert_eq!(uid.len(), 32); + assert!( + uid.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + ); + assert_eq!(uid, spans[0].span_context.trace_id().to_string()); + assert_eq!( + value["metadata"]["profiles"] + .as_array() + .unwrap() + .iter() + .filter(|p| *p == "trace") + .count(), + 1 + ); + assert_eq!(value["trace"].as_object().unwrap().len(), 1); +} + +#[test] +fn no_active_context_omits_trace_and_profile() { + let (value, spans) = capture(Sampler::AlwaysOn, false, None); + assert_no_trace(&value); + assert!(spans.is_empty()); +} + +#[test] +fn routed_emission_also_enriches_from_sampled_context() { + let (value, spans) = capture_route(Sampler::AlwaysOn, true, None, true); + assert_eq!( + value["trace"]["uid"], + spans[0].span_context.trace_id().to_string() + ); + assert!( + value["metadata"]["profiles"] + .as_array() + .unwrap() + .iter() + .any(|p| p == "trace") + ); +} + +#[test] +fn unsampled_active_context_omits_trace_and_profile() { + let (value, spans) = capture(Sampler::AlwaysOff, true, None); + assert_no_trace(&value); + assert!(spans.is_empty()); +} + +#[test] +fn explicit_trace_overrides_automatic_enrichment() { + let explicit = "0af7651916cd43dd8448eb211c80319c"; + let (value, spans) = capture( + Sampler::AlwaysOn, + true, + Some(TraceCorrelation { + uid: explicit.to_string(), + }), + ); + assert_eq!(value["trace"]["uid"], explicit); + assert_ne!(explicit, spans[0].span_context.trace_id().to_string()); + assert_eq!( + value["metadata"]["profiles"] + .as_array() + .unwrap() + .iter() + .filter(|p| *p == "trace") + .count(), + 1 + ); +} + +#[test] +fn active_tracing_span_without_otel_context_is_invalid() { + LazyLock::force(&SETUP); + tracing::subscriber::with_default(tracing_subscriber::registry(), || { + tracing::info_span!("no-otel-layer").in_scope(|| { + assert!(openshell_otel::current_ocsf_trace_correlation().is_none()); + }); + }); +} diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 5b2ce01c13..3f4c450a2c 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -666,6 +666,10 @@ async fn run_from_args( ) -> Result<()> { let prepared = prepare_server_config_with_drivers(&mut args, &matches, &compute_drivers)?; + openshell_ocsf::register_trace_correlation_extractor( + openshell_otel::current_ocsf_trace_correlation, + ); + // Initialize OCSF identity before tracing can emit gateway events. let gateway_identity = crate::gateway_ocsf::GatewayIdentity { name: prepared.config.name.clone(), diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index 6ac1239573..72d2295bf8 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -68,9 +68,11 @@ bundled-ca-roots = ["dep:webpki-roots"] openshell-sandbox-backend = { path = "../openshell-sandbox-backend", features = ["test-support"] } openshell-driver-vm = { path = "../openshell-driver-vm", default-features = false, features = ["allocation-tracking"] } +openshell-otel-test-support = { path = "../openshell-otel-test-support" } openshell-ocsf = { path = "../openshell-ocsf", features = ["test-support"] } openshell-otel = { path = "../openshell-otel" } openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } +opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic", "trace"] } opentelemetry_sdk = { workspace = true, features = ["testing"] } temp-env = "0.3" tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/openshell-supervisor-network/src/google_cloud_metadata.rs b/crates/openshell-supervisor-network/src/google_cloud_metadata.rs index 988f8073aa..73be442768 100644 --- a/crates/openshell-supervisor-network/src/google_cloud_metadata.rs +++ b/crates/openshell-supervisor-network/src/google_cloud_metadata.rs @@ -16,8 +16,10 @@ use http::StatusCode; use miette::{IntoDiagnostic, Result}; use openshell_core::provider_credentials::ProviderCredentialState; use openshell_ocsf::{ - ActivityId, HttpActivityBuilder, HttpRequest, SeverityId, StatusId, ocsf_emit, + ActionId, ActivityId, DispositionId, HttpActivityBuilder, HttpRequest, SeverityId, StatusId, }; + +use crate::telemetry::egress_emit; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; type MetadataResponse = (u16, &'static str, String); @@ -311,7 +313,7 @@ fn emit_metadata_event( status: StatusId, message: &str, ) { - ocsf_emit!(build_metadata_event( + egress_emit!(build_metadata_event( method, response_code, severity, @@ -327,7 +329,7 @@ fn build_metadata_event( status: StatusId, message: &str, ) -> openshell_ocsf::OcsfEvent { - HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + let mut builder = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::for_http_method(method)) .http_request(HttpRequest { http_method: method.parse().expect("HTTP method parsing is infallible"), @@ -338,8 +340,13 @@ fn build_metadata_event( }) .severity(severity) .status(status) - .message(message.to_string()) - .build() + .message(message.to_string()); + if matches!(response_code, 403 | 405) { + builder = builder + .action(ActionId::Denied) + .disposition(DispositionId::Blocked); + } + builder.build() } #[cfg(test)] @@ -347,6 +354,26 @@ mod tests { use super::*; use std::collections::HashMap; + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn metadata_rejections_have_deny_traces_but_operational_failures_do_not() { + use crate::telemetry::tests::{assert_deny_resolves, exported}; + + let (events, received) = exported("warn", || { + let ctx = make_context(HashMap::new()); + route_request(&ctx, "GET", PATH_TOKEN, &[]); + route_request(&ctx, "POST", PATH_TOKEN, &flavor_headers()); + route_request(&ctx, "GET", "/unknown", &flavor_headers()); + route_request(&ctx, "GET", PATH_TOKEN, &flavor_headers()); + }) + .await; + assert_eq!(events.len(), 4); + assert_deny_resolves(&events[..2], &received, true); + for event in &events[2..] { + assert!(event.get("trace").is_none()); + } + assert_eq!(received.spans.len(), 2); + } + fn token_binding() -> openshell_core::proto::StaticCredentialBinding { openshell_core::proto::StaticCredentialBinding { endpoints: vec![openshell_core::proto::StaticCredentialEndpointBinding { diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index d2344958b7..cbcf262777 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -5,10 +5,11 @@ use crate::l7::relay::L7EvalContext; use crate::opa::PolicyGenerationGuard; +use crate::telemetry::egress_emit; use miette::{Result, miette}; use openshell_ocsf::{ ActionId, ActivityId, DetectionFindingBuilder, DispositionId, Endpoint, FindingInfo, - HttpActivityBuilder, HttpRequest, NetworkActivityBuilder, SeverityId, StatusId, ocsf_emit, + HttpActivityBuilder, HttpRequest, NetworkActivityBuilder, SeverityId, StatusId, }; use std::path::PathBuf; use tokio::io::{AsyncRead, AsyncWrite}; @@ -168,24 +169,23 @@ pub fn emit_middleware_uninspectable(ctx: &L7EvalContext, detail: &str, denied: "Uninspectable traffic bypassed middleware (fail_open)" }) .build(); - ocsf_emit!(event); + egress_emit!(denied: denied, event); } pub fn emit_websocket_preflight_events( ctx: &L7EvalContext, outcome: &openshell_supervisor_middleware::WebSocketPreflightResult, ) { - emit_websocket_invocations(ctx, &outcome.invocations); - emit_websocket_coverage(ctx, &outcome.coverage); - for event in websocket_preflight_finding_events(outcome) { - ocsf_emit!(event); + let mut events = websocket_invocation_events(ctx, &outcome.invocations); + events.extend(websocket_coverage_events(ctx, &outcome.coverage)); + events.extend(websocket_preflight_finding_events(outcome)); + if outcome.session_capacity_exhausted { + events.push(middleware_session_capacity_exhausted_event(ctx)); } + crate::telemetry::emit_decision(events, None, !outcome.allowed); if outcome.saturated { emit_websocket_saturation(ctx); } - if outcome.session_capacity_exhausted { - emit_middleware_session_capacity_exhausted(ctx); - } } pub(super) fn emit_websocket_coverage( @@ -193,7 +193,7 @@ pub(super) fn emit_websocket_coverage( coverage: &[openshell_supervisor_middleware::WebSocketCoverage], ) { for event in websocket_coverage_events(ctx, coverage) { - ocsf_emit!(event); + egress_emit!(event); } } @@ -251,20 +251,23 @@ pub(super) fn emit_websocket_session_start_events( ctx: &L7EvalContext, outcome: &openshell_supervisor_middleware::WebSocketSessionStartOutcome, ) { - emit_websocket_invocations(ctx, &outcome.invocations); + crate::telemetry::emit_decision( + websocket_invocation_events(ctx, &outcome.invocations), + None, + !outcome.allowed, + ); } pub(super) fn emit_websocket_message_events( ctx: &L7EvalContext, outcome: &openshell_supervisor_middleware::WebSocketMessageOutcome, ) { - emit_websocket_invocations(ctx, &outcome.invocations); + let mut events = websocket_invocation_events(ctx, &outcome.invocations); + events.extend(websocket_message_finding_events(outcome)); + crate::telemetry::emit_decision(events, None, !outcome.allowed); if outcome.saturated { emit_websocket_saturation(ctx); } - for event in websocket_message_finding_events(outcome) { - ocsf_emit!(event); - } } pub(super) fn websocket_preflight_finding_events( @@ -311,10 +314,11 @@ pub(super) fn middleware_finding_events( .collect() } -fn emit_websocket_invocations( +fn websocket_invocation_events( ctx: &L7EvalContext, invocations: &[openshell_supervisor_middleware::WebSocketInvocation], -) { +) -> Vec { + let mut events = Vec::new(); for invocation in invocations { use openshell_supervisor_middleware::WebSocketInvocationOutcome as Outcome; let (action, disposition, severity, status, outcome_name) = match invocation.outcome { @@ -385,7 +389,7 @@ fn emit_websocket_invocations( invocation.transformed, )) .build(); - ocsf_emit!(event); + events.push(event); if invocation.failed { let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) .severity(if invocation.outcome == Outcome::FailClosed { @@ -406,7 +410,7 @@ fn emit_websocket_invocations( ]) .message("WebSocket middleware stage failed") .build(); - ocsf_emit!(event); + events.push(event); } if invocation.stage_disabled { let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) @@ -426,9 +430,10 @@ fn emit_websocket_invocations( "WebSocket middleware stage stream became unusable and was disabled for this session", ) .build(); - ocsf_emit!(event); + events.push(event); } } + events } fn emit_websocket_saturation(ctx: &L7EvalContext) { @@ -445,11 +450,11 @@ fn emit_websocket_saturation(ctx: &L7EvalContext) { ]) .message("WebSocket middleware work waited for admission capacity") .build(); - ocsf_emit!(event); + egress_emit!(event); } -fn emit_middleware_session_capacity_exhausted(ctx: &L7EvalContext) { - let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) +fn middleware_session_capacity_exhausted_event(ctx: &L7EvalContext) -> openshell_ocsf::OcsfEvent { + DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) .severity(SeverityId::Medium) .finding_info(FindingInfo::new( "openshell.middleware.session_capacity_exhausted", @@ -461,8 +466,7 @@ fn emit_middleware_session_capacity_exhausted(ctx: &L7EvalContext) { ("operation", "websocket_message"), ]) .message("Persistent middleware session admission was refused at process capacity") - .build(); - ocsf_emit!(event); + .build() } fn middleware_admission_exhausted_event(ctx: &L7EvalContext) -> openshell_ocsf::OcsfEvent { @@ -483,7 +487,7 @@ fn middleware_admission_exhausted_event(ctx: &L7EvalContext) -> openshell_ocsf:: } fn emit_middleware_admission_exhausted(ctx: &L7EvalContext) { - ocsf_emit!(middleware_admission_exhausted_event(ctx)); + egress_emit!(denied: true, middleware_admission_exhausted_event(ctx)); } /// Largest body-buffering limit across the entries that actually resolved to a @@ -811,7 +815,7 @@ fn emit_middleware_body_unavailable(ctx: &L7EvalContext, denied: bool) { "Request body exceeded middleware inspection cap; passed through (fail_open)" }) .build(); - ocsf_emit!(event); + egress_emit!(denied: denied, event); } /// Parse the raw header block into middleware-visible headers, preserving @@ -999,8 +1003,223 @@ fn emit_middleware_events( req: &crate::l7::provider::L7Request, outcome: &openshell_supervisor_middleware::ChainOutcome, ) { - for event in middleware_events(ctx, req, outcome) { - ocsf_emit!(event); + crate::telemetry::emit_decision(middleware_events(ctx, req, outcome), None, !outcome.allowed); +} + +#[cfg(test)] +mod trace_tests { + use super::*; + use crate::telemetry::tests::{assert_deny_resolves, exported}; + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn default_level_middleware_deny_finding_is_inside_exported_deny_span() { + let (events, received) = exported("warn", || { + let ctx = L7EvalContext { + host: "blocked.example".to_string(), + port: 443, + policy_name: "middleware-policy".to_string(), + ..Default::default() + }; + emit_middleware_uninspectable(&ctx, "unknown protocol", true); + }) + .await; + assert_eq!(events.len(), 1); + assert_eq!(events[0]["class_uid"], 2004); + assert_deny_resolves(&events, &received, true); + crate::telemetry::tests::assert_deny_attribute( + &received, + "openshell.policy.name", + "middleware-policy", + ); + crate::telemetry::tests::assert_deny_attribute( + &received, + "server.address", + "blocked.example", + ); + } + + fn finding() -> openshell_supervisor_middleware::NamespacedFinding { + openshell_supervisor_middleware::NamespacedFinding { + middleware: "guard".to_string(), + finding: openshell_core::proto::Finding { + r#type: "test.finding".to_string(), + label: "Test finding".to_string(), + count: 1, + severity: "high".to_string(), + ..Default::default() + }, + } + } + + fn assert_decision( + events: &[serde_json::Value], + received: &openshell_otel_test_support::ReceivedTraces, + denied: bool, + ) { + assert!(!events.is_empty()); + let deny_count = received + .spans + .iter() + .filter(|span| span.name == "supervisor.egress.deny") + .count(); + assert_eq!(deny_count, usize::from(denied)); + if denied { + assert_deny_resolves(events, received, true); + assert!( + events + .iter() + .all(|event| event["trace"]["uid"] == events[0]["trace"]["uid"]) + ); + } else { + assert!(events.iter().all(|event| event.get("trace").is_none())); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn http_chain_outcomes_group_findings_and_preserve_allowed_controls() { + for allowed in [true, false] { + let (events, received) = exported("warn", || { + let outcome = openshell_supervisor_middleware::ChainOutcome { + allowed, + reason: if allowed { + "allowed" + } else { + "middleware_failed: guard" + } + .to_string(), + body: vec![], + header_mutations: vec![], + findings: vec![finding()], + metadata: std::collections::BTreeMap::new(), + applied: vec![openshell_supervisor_middleware::MiddlewareInvocation { + name: "guard".to_string(), + implementation: "test/guard".to_string(), + decision: if allowed { + openshell_core::proto::Decision::Allow + } else { + openshell_core::proto::Decision::Deny + }, + transformed: false, + failed: false, + }], + denial: None, + }; + let request = crate::l7::provider::L7Request { + action: "POST".to_string(), + target: "/".to_string(), + query_params: std::collections::HashMap::new(), + raw_header: vec![], + body_length: crate::l7::provider::BodyLength::None, + }; + emit_middleware_events(&L7EvalContext::default(), &request, &outcome); + tracing::info_span!("test.export-barrier").in_scope(|| {}); + }) + .await; + assert!(events.len() >= 2); + assert_decision(&events, &received, !allowed); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn websocket_outcomes_group_domain_and_findings_with_fail_open_controls() { + use openshell_supervisor_middleware::WebSocketInvocationOutcome as Outcome; + for route in ["preflight", "message", "session-start"] { + for outcome in [ + Outcome::Allow, + Outcome::Deny, + Outcome::FailOpen, + Outcome::FailClosed, + ] { + let denied = matches!(outcome, Outcome::Deny | Outcome::FailClosed); + let (events, received) = exported("warn", || { + let ctx = L7EvalContext::default(); + let invocations = vec![openshell_supervisor_middleware::WebSocketInvocation { + config_name: "guard".to_string(), + implementation: "test/guard".to_string(), + outcome, + sequence: None, + original_size: 0, + replacement_size: None, + transformed: false, + failed: matches!(outcome, Outcome::FailOpen | Outcome::FailClosed), + stage_disabled: outcome == Outcome::FailOpen, + reason_code: None, + }]; + match route { + "preflight" => emit_websocket_preflight_events( + &ctx, + &openshell_supervisor_middleware::WebSocketPreflightResult { + allowed: !denied, + terminal_reason: None, + reason: "test".to_string(), + denial: None, + session: None, + findings: vec![finding()], + metadata: std::collections::BTreeMap::new(), + invocations, + coverage: vec![], + saturated: false, + session_capacity_exhausted: false, + }, + ), + "message" => emit_websocket_message_events( + &ctx, + &openshell_supervisor_middleware::WebSocketMessageOutcome { + allowed: !denied, + reason: "test".to_string(), + payload: String::new(), + findings: vec![finding()], + metadata: std::collections::BTreeMap::new(), + invocations, + denial: None, + saturated: false, + platform_oversize: false, + }, + ), + _ => emit_websocket_session_start_events( + &ctx, + &openshell_supervisor_middleware::WebSocketSessionStartOutcome { + allowed: !denied, + terminal_reason: None, + reason: "test".to_string(), + invocations, + }, + ), + } + tracing::info_span!("test.export-barrier").in_scope(|| {}); + }) + .await; + assert_decision(&events, &received, denied); + } + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn session_capacity_only_traces_fail_closed_refusals() { + for allowed in [true, false] { + let (events, received) = exported("warn", || { + emit_websocket_preflight_events( + &L7EvalContext::default(), + &openshell_supervisor_middleware::WebSocketPreflightResult { + allowed, + terminal_reason: None, + reason: "middleware_session_capacity_exhausted".to_string(), + denial: None, + session: None, + findings: vec![], + metadata: std::collections::BTreeMap::new(), + invocations: vec![], + coverage: vec![], + saturated: false, + session_capacity_exhausted: true, + }, + ); + tracing::info_span!("test.export-barrier").in_scope(|| {}); + }) + .await; + assert_eq!(events.len(), 1); + assert_decision(&events, &received, !allowed); + } } } diff --git a/crates/openshell-supervisor-network/src/l7/mod.rs b/crates/openshell-supervisor-network/src/l7/mod.rs index 3f26114998..295eabf234 100644 --- a/crates/openshell-supervisor-network/src/l7/mod.rs +++ b/crates/openshell-supervisor-network/src/l7/mod.rs @@ -386,7 +386,7 @@ pub fn parse_l7_config(val: ®orus::Value) -> Option { "rejecting endpoint: unrecognized credential_signing value {other:?}" )) .build(); - openshell_ocsf::ocsf_emit!(event); + crate::telemetry::egress_emit!(event); return None; } _ => CredentialSigning::None, @@ -402,7 +402,7 @@ pub fn parse_l7_config(val: ®orus::Value) -> Option { .state(openshell_ocsf::StateId::Disabled, "invalid") .message("rejecting endpoint: credential_signing requires signing_service".to_string()) .build(); - openshell_ocsf::ocsf_emit!(event); + crate::telemetry::egress_emit!(event); return None; } @@ -429,8 +429,12 @@ pub fn parse_l7_config(val: ®orus::Value) -> Option { }) } -pub(crate) fn emit_uninspected_credential_finding(host: &str, policy_name: &str, surface: &str) { - let event = openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) +pub(crate) fn build_uninspected_credential_finding( + host: &str, + policy_name: &str, + surface: &str, +) -> openshell_ocsf::OcsfEvent { + openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) .severity(openshell_ocsf::SeverityId::High) .finding_info(openshell_ocsf::FindingInfo::new( "openshell.credentials.traffic_uninspectable", @@ -443,8 +447,7 @@ pub(crate) fn emit_uninspected_credential_finding(host: &str, policy_name: &str, ("disposition", "denied"), ]) .message("Uninspected credential-bearing traffic denied") - .build(); - openshell_ocsf::ocsf_emit!(event); + .build() } impl L7EndpointConfig { diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index f3e9e34741..0ae1ba8893 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -20,6 +20,7 @@ use crate::l7::provider::{L7Provider, RelayOutcome}; use crate::l7::rest::WebSocketExtensionMode; use crate::l7::{EndpointObserver, EnforcementMode, L7EndpointConfig, L7Protocol, L7RequestInfo}; use crate::opa::{PolicyGenerationGuard, TunnelPolicyEngine}; +use crate::telemetry::egress_emit; use miette::{IntoDiagnostic, Result, miette}; use openshell_core::activity::{ActivitySender, try_record_activity}; use openshell_core::endpoint_status::{EndpointObservationSender, EndpointResult}; @@ -27,7 +28,6 @@ use openshell_core::secrets::{self, SecretResolver}; use openshell_ocsf::{ ActionId, ActivityId, DetectionFindingBuilder, DispositionId, Endpoint, FindingInfo, HttpActivityBuilder, HttpRequest, NetworkActivityBuilder, SeverityId, StatusId, Url as OcsfUrl, - ocsf_emit, }; #[cfg(test)] use std::collections::BTreeMap; @@ -287,8 +287,14 @@ where .into_diagnostic()?; client.flush().await.into_diagnostic()?; - ocsf_emit!(build_request_authority_mismatch_event(ctx, method)); - ocsf_emit!(build_request_authority_mismatch_finding(ctx)); + crate::telemetry::emit_decision( + [ + build_request_authority_mismatch_event(ctx, method), + build_request_authority_mismatch_finding(ctx), + ], + None, + false, + ); Ok(()) } @@ -338,7 +344,7 @@ where crate::l7::mcp::selected_revision_context(request, version) ); let summary = l7_protocol_log_summary(None, Some(&info)); - ocsf_emit!(build_l7_request_event( + egress_emit!(build_l7_request_event( ctx, &request.action, redacted_target, @@ -419,7 +425,7 @@ where } let reason = error.rejection_detail(request); let summary = l7_protocol_log_summary(None, Some(info)); - ocsf_emit!(build_l7_request_event( + egress_emit!(build_l7_request_event( ctx, &request.action, redacted_target, @@ -613,15 +619,9 @@ where .into_diagnostic()?; client.flush().await.into_diagnostic()?; - ocsf_emit!(build_credential_resolution_event( - ctx, - method, - endpoint_mismatch - )); - - if endpoint_mismatch { - ocsf_emit!(build_credential_endpoint_mismatch_finding(ctx)); - } + let event = build_credential_resolution_event(ctx, method, endpoint_mismatch); + let finding = endpoint_mismatch.then(|| build_credential_endpoint_mismatch_finding(ctx)); + crate::telemetry::emit_decision(std::iter::once(event).chain(finding), None, false); Ok(()) } @@ -1015,7 +1015,7 @@ fn emit_parse_rejection(ctx: &L7EvalContext, detail: &str, engine_type: &str) { )) .status_detail(detail) .build(); - ocsf_emit!(event); + egress_emit!(event); emit_activity(ctx, true, "l7_parse_rejection"); } @@ -1131,7 +1131,7 @@ where .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) .message("SQL L7 provider not yet implemented, falling back to passthrough") .build(); - ocsf_emit!(event); + egress_emit!(event); } tokio::io::copy_bidirectional(client, upstream) .await @@ -1664,7 +1664,7 @@ fn emit_l7_request_log( reason, protocol_summary, ); - ocsf_emit!(event); + egress_emit!(event); emit_activity(ctx, decision_str == "deny", "l7_policy"); } @@ -1830,7 +1830,7 @@ where } else { "raw bidirectional relay (L7 enforcement no longer active)" }; - ocsf_emit!( + egress_emit!( NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Other) .activity_name("Upgrade") @@ -2169,7 +2169,7 @@ where request_info.action, ctx.host, ctx.port, redacted_target, reason, )) .build(); - ocsf_emit!(event); + egress_emit!(event); } if allowed || config.enforcement == EnforcementMode::Audit { @@ -2377,7 +2377,7 @@ pub(crate) fn emit_policy_reload( port: u16, policy_name: &str, ) { - ocsf_emit!( + egress_emit!( NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Open) .action(ActionId::Denied) @@ -2597,7 +2597,7 @@ where &reason, )) .build(); - ocsf_emit!(event); + egress_emit!(event); } if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { @@ -2879,7 +2879,7 @@ where request_info.action, ctx.host, ctx.port, redacted_target, reason, )) .build(); - ocsf_emit!(event); + egress_emit!(event); } if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { @@ -3350,7 +3350,7 @@ fn emit_transformed_body_decision( request_info.action, ctx.host, ctx.port, request_info.target, reason )) .build(); - ocsf_emit!(event); + egress_emit!(event); } fn jsonrpc_policy_input(info: &crate::l7::jsonrpc::JsonRpcRequestInfo) -> serde_json::Value { @@ -3583,7 +3583,7 @@ where req.action, ctx.host, ctx.port, redacted_target, )) .build(); - ocsf_emit!(event); + egress_emit!(event); } let request_id = uuid::Uuid::new_v4().to_string(); @@ -3755,6 +3755,63 @@ where Ok(()) } +#[cfg(test)] +mod trace_tests { + use super::*; + use crate::telemetry::tests::{assert_deny_resolves, exported}; + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn authority_mismatch_domain_and_finding_share_one_trace() { + let (events, received) = exported("warn", || { + let ctx = L7EvalContext { + host: "blocked.example".to_string(), + port: 443, + ..Default::default() + }; + let mut response = Vec::new(); + futures::executor::block_on(reject_request_authority_mismatch( + &mut response, + &ctx, + "POST", + )) + .unwrap(); + assert!(response.starts_with(b"HTTP/1.1 403 Forbidden")); + }) + .await; + assert_eq!(events.len(), 2); + assert_eq!(received.spans.len(), 1); + assert_eq!(events[0]["trace"]["uid"], events[1]["trace"]["uid"]); + assert_deny_resolves(&events, &received, true); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn default_level_l7_relay_deny_uid_resolves_to_one_span_root_trace() { + let (events, received) = exported("warn", || { + let ctx = L7EvalContext { + host: "blocked.example".to_string(), + port: 443, + policy_name: "rest-policy".to_string(), + ..Default::default() + }; + emit_parse_rejection(&ctx, "ambiguous framing", "l7-rest"); + }) + .await; + assert_eq!(events.len(), 1); + assert_deny_resolves(&events, &received, true); + assert_eq!(received.spans.len(), 1); + crate::telemetry::tests::assert_deny_attribute( + &received, + "openshell.policy.name", + "rest-policy", + ); + crate::telemetry::tests::assert_deny_attribute( + &received, + "server.address", + "blocked.example", + ); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 7b85942666..acbadc0297 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -1184,7 +1184,7 @@ where port = options.port, )) .build(); - openshell_ocsf::ocsf_emit!(event); + crate::telemetry::egress_emit!(event); } _ => { return Err(miette::Report::new(CredentialUnavailableError::new( @@ -1534,7 +1534,6 @@ fn emit_uninspected_body_credential_denial( req.action, options.host, options.port )) .build(); - openshell_ocsf::ocsf_emit!(event); let finding = openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) .severity(openshell_ocsf::SeverityId::High) .finding_info(openshell_ocsf::FindingInfo::new( @@ -1548,7 +1547,7 @@ fn emit_uninspected_body_credential_denial( ]) .message("Request body credential placeholder denied") .build(); - openshell_ocsf::ocsf_emit!(finding); + crate::telemetry::emit_decision([event, finding], None, false); } struct PreparedRequestBody { @@ -6916,6 +6915,238 @@ mod tests { (outcome, delivered) } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn response_trace_failure_records_share_the_terminal_decision() { + use crate::telemetry::tests::{assert_deny_resolves, exported}; + + // This relay fixture has no CONNECT span, so denies are roots at both + // levels. The thread-local export harness observes the relay task; + // emissions in spawned tasks would need explicit subscriber propagation. + for level in ["warn", "debug"] { + for (method, script) in [ + ("GET", ResponseRelayScript::BlockStream), + ("GET", ResponseRelayScript::InvalidBodySequence), + ("GET", ResponseRelayScript::InvalidWholeBodySequence), + ("HEAD", ResponseRelayScript::WholeBody), + ] { + let (events, received) = exported(level, || { + let (outcome, delivered) = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(run_response_middleware_relay( + b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello", + method, + script, + )) + }); + if method == "HEAD" + || matches!(script, ResponseRelayScript::InvalidWholeBodySequence) + { + assert!(matches!(outcome.unwrap(), RelayOutcome::Consumed)); + assert!(delivered.starts_with(b"HTTP/1.1 502 Bad Gateway\r\n")); + } else { + assert!(outcome.is_err()); + assert!(delivered.starts_with(b"HTTP/1.1 200 OK\r\n")); + assert!(!String::from_utf8_lossy(&delivered).contains("502 Bad Gateway")); + } + }) + .await; + let terminal = events + .iter() + .find(|event| { + event["finding_info"]["uid"] == "openshell.middleware.http_response_failure" + }) + .expect("terminal delivery failure is recorded"); + let companions: Vec<_> = events + .iter() + .filter(|event| { + event.get("finding_info").is_some() + || event["unmapped"]["failed"] == true + || event["action_id"] == openshell_ocsf::ActionId::Denied as u8 + }) + .cloned() + .collect(); + assert!(companions.len() >= 2, "failure includes diagnostic records"); + assert_eq!( + received + .spans + .iter() + .filter(|span| span.name == "supervisor.egress.deny") + .count(), + 1, + "one response stop is one exported decision" + ); + assert_deny_resolves(&companions, &received, true); + assert!( + companions + .iter() + .all(|event| event["trace"]["uid"] == terminal["trace"]["uid"]) + ); + } + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn response_trace_upstream_io_failure_is_not_an_enforced_denial() { + struct FailedReader; + impl AsyncRead for FailedReader { + fn poll_read( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + _buf: &mut ReadBuf<'_>, + ) -> Poll> { + Poll::Ready(Err(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "upstream response interrupted", + ))) + } + } + + for script in [ResponseRelayScript::Stream, ResponseRelayScript::WholeBody] { + let (events, received) = crate::telemetry::tests::exported("warn", || { + tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(async { + let (runner, chain) = response_middleware_fixture(script); + let mut upstream = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\n" + .as_slice() + .chain(FailedReader); + let mut delivered = Vec::new(); + let outcome = relay_response( + "GET", + &mut upstream, + &mut delivered, + RelayResponseOptions::default(), + Some(response_middleware_context(&runner, &chain, "GET")), + ) + .await; + if matches!(script, ResponseRelayScript::Stream) { + assert!(outcome.is_err()); + assert!(delivered.starts_with(b"HTTP/1.1 200 OK\r\n")); + assert!( + !String::from_utf8_lossy(&delivered).contains("502 Bad Gateway") + ); + } else { + assert!(matches!(outcome.unwrap(), RelayOutcome::Consumed)); + assert!(delivered.starts_with(b"HTTP/1.1 502 Bad Gateway\r\n")); + } + }); + }); + tracing::info_span!("test.export-barrier").in_scope(|| {}); + }) + .await; + assert!(events.iter().any(|event| event["finding_info"]["uid"] + == "openshell.middleware.http_response_failure")); + assert!(events.iter().all(|event| event.get("trace").is_none())); + assert!( + !received + .spans + .iter() + .any(|span| span.name == "supervisor.egress.deny") + ); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn response_trace_downstream_body_failure_is_not_an_enforced_denial() { + #[derive(Default)] + struct DisconnectAfterHead(Vec); + impl AsyncWrite for DisconnectAfterHead { + fn poll_write( + mut self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + if self.0.ends_with(b"\r\n\r\n") { + return Poll::Ready(Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "client disconnected", + ))); + } + self.0.extend_from_slice(buf); + Poll::Ready(Ok(buf.len())) + } + fn poll_flush( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn poll_shutdown( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + } + + let (events, received) = crate::telemetry::tests::exported("warn", || { + tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(async { + let (runner, chain) = response_middleware_fixture(ResponseRelayScript::Stream); + let mut upstream = + b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello".as_slice(); + let mut delivered = DisconnectAfterHead::default(); + let outcome = relay_response( + "GET", + &mut upstream, + &mut delivered, + RelayResponseOptions::default(), + Some(response_middleware_context(&runner, &chain, "GET")), + ) + .await; + assert!(outcome.is_err()); + assert!(delivered.0.starts_with(b"HTTP/1.1 200 OK\r\n")); + assert!(!String::from_utf8_lossy(&delivered.0).contains("502 Bad Gateway")); + }); + }); + tracing::info_span!("test.export-barrier").in_scope(|| {}); + }) + .await; + assert!( + events.iter().any(|event| event["finding_info"]["uid"] + == "openshell.middleware.http_response_failure") + ); + assert!(events.iter().all(|event| event.get("trace").is_none())); + assert!( + !received + .spans + .iter() + .any(|span| span.name == "supervisor.egress.deny") + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn response_trace_allowed_and_fail_open_controls_do_not_create_deny_spans() { + for script in [ + ResponseRelayScript::Stream, + ResponseRelayScript::InvalidBodySequence, + ResponseRelayScript::InvalidWholeBodySequence, + ] { + let (events, received) = crate::telemetry::tests::exported("warn", || { + let (outcome, delivered) = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on( + run_response_middleware_relay_with_error( + b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello", + "GET", + script, + openshell_supervisor_middleware::OnError::FailOpen, + ), + ) + }); + assert!(outcome.is_ok()); + assert!(delivered.starts_with(b"HTTP/1.1 200 OK\r\n")); + tracing::info_span!("test.export-barrier").in_scope(|| {}); + }) + .await; + assert!(!events.is_empty()); + assert!(events.iter().all(|event| event.get("trace").is_none())); + assert!( + !received + .spans + .iter() + .any(|span| span.name == "supervisor.egress.deny") + ); + } + } + #[tokio::test] async fn response_middleware_headers_only_mutates_head_and_preserves_framing() { let (outcome, delivered) = run_response_middleware_relay( diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs index 3efc323bdb..9b54ea9993 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs @@ -408,16 +408,26 @@ where allowed = preflight.allowed, "HTTP response middleware preflight completed" ); - for event in crate::l7::middleware::middleware_finding_events(&preflight.findings) { - openshell_ocsf::ocsf_emit!(event); - } + let mut preflight_events = + crate::l7::middleware::middleware_finding_events(&preflight.findings); if !preflight.allowed { - emit_http_response_middleware_invocations( + preflight_events.extend(http_response_decision_events( middleware.policy_name, &middleware.target, status_code, &preflight.invocations, - ); + )); + if preflight.denial.is_none() { + preflight_events.push(http_response_middleware_failure_event( + middleware.policy_name, + &middleware.target, + status_code, + false, + )); + } + } + crate::telemetry::emit_decision(preflight_events, None, !preflight.allowed); + if !preflight.allowed { if let Some(denial) = preflight.denial.as_ref() { send_response_middleware_denial( client, @@ -428,12 +438,6 @@ where ) .await?; } else { - emit_http_response_middleware_failure( - middleware.policy_name, - &middleware.target, - status_code, - false, - ); send_response_delivery_failure( client, request_method, @@ -487,6 +491,8 @@ where &middleware.target, status_code, &error.diagnostics, + true, + Some(false), ); send_response_delivery_failure( client, @@ -509,7 +515,7 @@ where &finish.invocations, ); for event in crate::l7::middleware::middleware_finding_events(&finish.findings) { - openshell_ocsf::ocsf_emit!(event); + crate::telemetry::egress_emit!(event); } if finish.strip_stale_integrity_headers { strip_response_integrity_headers(&mut headers); @@ -610,13 +616,21 @@ where openshell_core::proto::MiddlewareSessionEndReason::UpstreamDisconnect }; session.end(end_reason).await; + // Diagnostics from a middleware stop and its terminal finding are + // one decision. Transport/framing errors still record delivery + // failure, but do not imply an enforced middleware refusal. + let empty_diagnostics = + openshell_supervisor_middleware::HttpResponseDiagnostics::default(); + emit_http_response_diagnostics( + middleware.policy_name, + &middleware.target, + status_code, + middleware_stop.map_or(&empty_diagnostics, |stop| &stop.diagnostics), + middleware_stop.is_some(), + (committed || middleware_stop.is_none_or(|stop| stop.failure.denial.is_none())) + .then_some(committed), + ); if committed { - emit_http_response_middleware_failure( - middleware.policy_name, - &middleware.target, - status_code, - true, - ); return Err(error); } debug!(error = %error, "HTTP response processing failed before commitment"); @@ -630,12 +644,6 @@ where ) .await?; } else { - emit_http_response_middleware_failure( - middleware.policy_name, - &middleware.target, - status_code, - false, - ); send_response_delivery_failure( client, request_method, @@ -665,14 +673,10 @@ where &middleware.target, status_code, &error.diagnostics, + true, + (committed || error.denial.is_none()).then_some(committed), ); if committed { - emit_http_response_middleware_failure( - middleware.policy_name, - &middleware.target, - status_code, - true, - ); return Err(miette!( "HTTP response middleware failed after commitment: {error}" )); @@ -688,12 +692,6 @@ where ) .await?; } else { - emit_http_response_middleware_failure( - middleware.policy_name, - &middleware.target, - status_code, - false, - ); send_response_delivery_failure( client, request_method, @@ -715,7 +713,7 @@ where &finish.invocations, ); for event in crate::l7::middleware::middleware_finding_events(&finish.findings) { - openshell_ocsf::ocsf_emit!(event); + crate::telemetry::egress_emit!(event); } if whole_body && !committed { @@ -861,29 +859,40 @@ where }) } +fn http_response_decision_events( + policy_name: &str, + target: &HttpRequestTarget, + status_code: u16, + invocations: &[openshell_supervisor_middleware::HttpResponseInvocation], +) -> Vec { + let mut events = + http_response_middleware_invocation_events(policy_name, target, status_code, invocations); + for invocation in invocations { + events.extend(http_response_middleware_fail_open_finding_event( + policy_name, + target, + invocation, + )); + events.extend(http_response_middleware_block_finding_event( + policy_name, + target, + invocation, + )); + } + events +} + fn emit_http_response_middleware_invocations( policy_name: &str, target: &HttpRequestTarget, status_code: u16, invocations: &[openshell_supervisor_middleware::HttpResponseInvocation], ) { - for event in - http_response_middleware_invocation_events(policy_name, target, status_code, invocations) - { - openshell_ocsf::ocsf_emit!(event); - } - for invocation in invocations { - if let Some(event) = - http_response_middleware_fail_open_finding_event(policy_name, target, invocation) - { - openshell_ocsf::ocsf_emit!(event); - } - if let Some(event) = - http_response_middleware_block_finding_event(policy_name, target, invocation) - { - openshell_ocsf::ocsf_emit!(event); - } - } + crate::telemetry::emit_decision( + http_response_decision_events(policy_name, target, status_code, invocations), + None, + false, + ); } fn emit_http_response_diagnostics( @@ -891,16 +900,23 @@ fn emit_http_response_diagnostics( target: &HttpRequestTarget, status_code: u16, diagnostics: &openshell_supervisor_middleware::HttpResponseDiagnostics, + denied: bool, + terminal_failure: Option, ) { - emit_http_response_middleware_invocations( - policy_name, - target, - status_code, - &diagnostics.invocations, - ); - for event in crate::l7::middleware::middleware_finding_events(&diagnostics.findings) { - openshell_ocsf::ocsf_emit!(event); + let mut events = + http_response_decision_events(policy_name, target, status_code, &diagnostics.invocations); + events.extend(crate::l7::middleware::middleware_finding_events( + &diagnostics.findings, + )); + if let Some(committed) = terminal_failure { + events.push(http_response_middleware_failure_event( + policy_name, + target, + status_code, + committed, + )); } + crate::telemetry::emit_decision(events, None, denied); } pub(super) fn http_response_middleware_invocation_events( @@ -1068,8 +1084,26 @@ fn emit_http_response_middleware_failure( status_code: u16, committed: bool, ) { + crate::telemetry::emit_decision( + [http_response_middleware_failure_event( + policy_name, + target, + status_code, + committed, + )], + None, + true, + ); +} + +fn http_response_middleware_failure_event( + policy_name: &str, + target: &HttpRequestTarget, + status_code: u16, + committed: bool, +) -> openshell_ocsf::OcsfEvent { let status_code = status_code.to_string(); - let event = openshell_ocsf::DetectionFindingBuilder::new(ocsf_ctx()) + openshell_ocsf::DetectionFindingBuilder::new(ocsf_ctx()) .severity(openshell_ocsf::SeverityId::High) .finding_info(openshell_ocsf::FindingInfo::new( "openshell.middleware.http_response_failure", @@ -1093,8 +1127,7 @@ fn emit_http_response_middleware_failure( } else { "HTTP response middleware failed before response commitment" }) - .build(); - openshell_ocsf::ocsf_emit!(event); + .build() } #[derive(Debug)] @@ -1627,11 +1660,18 @@ struct ResponseOutputState<'a> { #[derive(Debug)] struct ResponseMiddlewareStop { failure: openshell_supervisor_middleware::HttpResponseMiddlewareFailure, + diagnostics: openshell_supervisor_middleware::HttpResponseDiagnostics, } impl ResponseMiddlewareStop { - fn new(failure: openshell_supervisor_middleware::HttpResponseMiddlewareFailure) -> Self { - Self { failure } + fn new( + failure: openshell_supervisor_middleware::HttpResponseMiddlewareFailure, + diagnostics: openshell_supervisor_middleware::HttpResponseDiagnostics, + ) -> Self { + Self { + failure, + diagnostics, + } } } @@ -1676,13 +1716,17 @@ async fn expire_whole_body_deadline( guard.ensure_current()?; } let diagnostics = session.take_diagnostics(); - emit_http_response_diagnostics( - framing.policy_name, - framing.target, - framing.status_code, - &diagnostics, - ); - let output = output.map_err(ResponseMiddlewareStop::new)?; + if output.is_ok() { + emit_http_response_diagnostics( + framing.policy_name, + framing.target, + framing.status_code, + &diagnostics, + false, + None, + ); + } + let output = output.map_err(|failure| ResponseMiddlewareStop::new(failure, diagnostics))?; deliver_response_units(client, output, framing, session.requires_whole_body()).await } @@ -1818,13 +1862,17 @@ async fn process_response_unit( guard.ensure_current()?; } let diagnostics = session.take_diagnostics(); - emit_http_response_diagnostics( - framing.policy_name, - framing.target, - framing.status_code, - &diagnostics, - ); - let output = output.map_err(ResponseMiddlewareStop::new)?; + if output.is_ok() { + emit_http_response_diagnostics( + framing.policy_name, + framing.target, + framing.status_code, + &diagnostics, + false, + None, + ); + } + let output = output.map_err(|failure| ResponseMiddlewareStop::new(failure, diagnostics))?; deliver_response_units(client, output, framing, session.requires_whole_body()).await } @@ -2081,6 +2129,131 @@ pub(super) fn response_is_event_stream(headers: &str) -> bool { #[cfg(test)] mod tests { + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn response_denial_and_companion_findings_share_one_trace() { + use crate::telemetry::tests::{assert_deny_resolves, exported}; + use openshell_supervisor_middleware::HttpResponseInvocationOutcome; + + let (events, received) = exported("warn", || { + let diagnostics = openshell_supervisor_middleware::HttpResponseDiagnostics { + invocations: vec![openshell_supervisor_middleware::HttpResponseInvocation { + config_name: "guard".to_string(), + implementation: "test/guard".to_string(), + outcome: HttpResponseInvocationOutcome::BlockDelivery, + sequence: None, + input_size: 0, + output_size: None, + failed: false, + stage_disabled: false, + reason_code: None, + failure_category: None, + }], + findings: vec![openshell_supervisor_middleware::NamespacedFinding { + middleware: "guard".to_string(), + finding: openshell_core::proto::Finding { + r#type: "test.finding".to_string(), + label: "Test finding".to_string(), + count: 1, + severity: "high".to_string(), + ..Default::default() + }, + }], + ..Default::default() + }; + super::emit_http_response_diagnostics( + "response-policy", + &super::HttpRequestTarget::default(), + 200, + &diagnostics, + true, + None, + ); + }) + .await; + assert!(events.len() >= 2); + assert_eq!(received.spans.len(), 1); + assert!( + events + .iter() + .all(|event| event["trace"]["uid"] == events[0]["trace"]["uid"]) + ); + assert_deny_resolves(&events, &received, true); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn response_terminal_failure_without_diagnostics_still_exports_a_deny_trace() { + use crate::telemetry::tests::{assert_deny_resolves, exported}; + + // Exercise the emission boundary directly: platform refusals can lack + // invocation diagnostics, but must retain their terminal failure record. + for committed in [false, true] { + let (events, received) = exported("warn", || { + super::emit_http_response_diagnostics( + "response-policy", + &super::HttpRequestTarget::default(), + 200, + &openshell_supervisor_middleware::HttpResponseDiagnostics::default(), + true, + Some(committed), + ); + }) + .await; + + assert_eq!(events.len(), 1, "only the terminal failure is emitted"); + assert_eq!( + events[0]["finding_info"]["uid"], + "openshell.middleware.http_response_failure" + ); + assert_eq!(received.spans.len(), 1); + assert_deny_resolves(&events, &received, true); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn response_diagnostic_findings_create_deny_spans_only_when_delivery_is_denied() { + use crate::telemetry::tests::{assert_deny_resolves, exported}; + + for denied in [false, true] { + let (events, received) = exported("warn", || { + let diagnostics = openshell_supervisor_middleware::HttpResponseDiagnostics { + findings: vec![openshell_supervisor_middleware::NamespacedFinding { + middleware: "content-guard".into(), + finding: openshell_core::proto::Finding { + r#type: "test.finding".into(), + label: "Test finding".into(), + count: 1, + severity: "high".into(), + ..Default::default() + }, + }], + ..Default::default() + }; + super::emit_http_response_diagnostics( + "response-policy", + &super::HttpRequestTarget::default(), + 200, + &diagnostics, + denied, + None, + ); + tracing::info_span!("test.export-barrier").in_scope(|| {}); + }) + .await; + assert_eq!(events.len(), 1); + if denied { + assert_deny_resolves(&events, &received, true); + } else { + assert!(events[0].get("trace").is_none()); + assert!( + !received + .spans + .iter() + .any(|span| span.name == "supervisor.egress.deny") + ); + } + } + } + use super::{ ResponseFraming, parse_connection_keep_alive, parse_response_head_for_middleware, response_is_http_10, serialize_response_head, diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs index 65d7b70d08..f868575097 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs @@ -22,12 +22,13 @@ use openshell_core::token_grant_failure::{ }; use openshell_ocsf::{ ActionId, ActivityId, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, SeverityId, - StatusId, ctx::ctx as ocsf_ctx, ocsf_emit, + StatusId, ctx::ctx as ocsf_ctx, }; use tracing::warn; use crate::l7::provider::L7Request; use crate::l7::relay::L7EvalContext; +use crate::telemetry::egress_emit; /// Borrowed grant configuration for one endpoint-bound provider credential. pub struct TokenGrantRequest<'a> { @@ -340,7 +341,7 @@ fn report_refused_grant( .map_or_else(|| "none".to_string(), |status| status.to_string()), "Token grant failed" ); - ocsf_emit!( + egress_emit!( HttpActivityBuilder::new(ocsf_ctx()) .activity(ActivityId::Fail) .action(ActionId::Denied) @@ -398,7 +399,7 @@ async fn inject_selected( inject_header(&raw_header, &name, &value).map_err(TokenGrantInjectionError::new)?; } for (provider_key, _) in credentials { - ocsf_emit!( + egress_emit!( HttpActivityBuilder::new(ocsf_ctx()) .activity(ActivityId::Other) .action(ActionId::Allowed) diff --git a/crates/openshell-supervisor-network/src/l7/websocket.rs b/crates/openshell-supervisor-network/src/l7/websocket.rs index f2ea281a81..171b2c5670 100644 --- a/crates/openshell-supervisor-network/src/l7/websocket.rs +++ b/crates/openshell-supervisor-network/src/l7/websocket.rs @@ -9,13 +9,13 @@ use crate::l7::relay::{L7EvalContext, evaluate_l7_request}; use crate::l7::{EnforcementMode, L7RequestInfo}; use crate::opa::{PolicyGenerationGuard, TunnelPolicyEngine}; +use crate::telemetry::egress_emit; use flate2::{Compress, Compression, Decompress, FlushCompress, FlushDecompress, Status}; use miette::{IntoDiagnostic, Result, miette}; use openshell_core::provider_credentials::ProviderCredentialState; use openshell_core::secrets::{SecretResolver, contains_reserved_credential_marker}; use openshell_ocsf::{ ActionId, ActivityId, DispositionId, Endpoint, NetworkActivityBuilder, SeverityId, StatusId, - ocsf_emit, }; use std::collections::HashMap; use std::future::Future; @@ -620,27 +620,26 @@ async fn write_policy_violation_close(writer: &mut W) -> } fn emit_credential_endpoint_mismatch(host: &str, port: u16, policy_name: &str) { - ocsf_emit!( - NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::High) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(host, port)) - .firewall_rule(policy_name, "credential-binding") - .message(format!( - "WebSocket credential use denied: credential is not authorized for {host}:{port}" - )) - .status_detail("credential_endpoint_mismatch") - .build() - ); - ocsf_emit!(crate::l7::build_credential_endpoint_mismatch_finding( + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::High) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(host, port)) + .firewall_rule(policy_name, "credential-binding") + .message(format!( + "WebSocket credential use denied: credential is not authorized for {host}:{port}" + )) + .status_detail("credential_endpoint_mismatch") + .build(); + let finding = crate::l7::build_credential_endpoint_mismatch_finding( policy_name, host, Some("websocket"), "Provider credential endpoint binding mismatch; WebSocket closed", - )); + ); + crate::telemetry::emit_decision([event, finding], None, false); } async fn relay_client_to_server( @@ -1386,8 +1385,31 @@ fn emit_uninspected_credential_denial(host: &str, port: u16, policy_name: &str, "WebSocket credential traffic denied for {host}:{port}" )) .build(); - ocsf_emit!(event); - crate::l7::emit_uninspected_credential_finding(host, policy_name, surface); + let finding = crate::l7::build_uninspected_credential_finding(host, policy_name, surface); + crate::telemetry::emit_decision([event, finding], None, false); +} + +#[cfg(test)] +mod trace_tests { + use super::*; + use crate::telemetry::tests::{assert_deny_resolves, exported}; + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn paired_websocket_credential_denial_shares_one_deny_span() { + let (events, received) = exported("warn", || { + emit_uninspected_credential_denial( + "blocked.example", + 443, + "credential-policy", + "websocket", + ); + }) + .await; + assert_eq!(events.len(), 2); + assert_eq!(received.spans.len(), 1); + assert_eq!(events[0]["trace"]["uid"], events[1]["trace"]["uid"]); + assert_deny_resolves(&events, &received, true); + } } fn inspect_websocket_text_message( @@ -2061,7 +2083,7 @@ fn emit_rewrite_event(host: &str, port: u16, policy_name: &str, replacements: us .firewall_rule(policy_name, "l7-websocket") .message(rewrite_event_message(host, port, replacements)) .build(); - ocsf_emit!(event); + egress_emit!(event); } fn rewrite_event_message(host: &str, port: u16, replacements: usize) -> String { @@ -2114,7 +2136,7 @@ fn emit_websocket_l7_event( request_info.action, request_info.target, summary )) .build(); - ocsf_emit!(event); + egress_emit!(event); } fn observe_termination( @@ -2127,7 +2149,7 @@ fn observe_termination( emit_protocol_failure(host, port, policy_name, failure_class); } if termination.cause == WebSocketTerminationCause::CapacityExhausted { - ocsf_emit!( + egress_emit!( NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Open) .action(ActionId::Denied) @@ -2151,7 +2173,7 @@ fn emit_protocol_failure( policy_name: &str, failure_class: FrameFailureClass, ) { - ocsf_emit!(protocol_failure_event( + egress_emit!(protocol_failure_event( host, port, policy_name, diff --git a/crates/openshell-supervisor-network/src/lib.rs b/crates/openshell-supervisor-network/src/lib.rs index d9fcd004b6..74a6698b92 100644 --- a/crates/openshell-supervisor-network/src/lib.rs +++ b/crates/openshell-supervisor-network/src/lib.rs @@ -24,6 +24,7 @@ pub mod proxy; pub mod run; pub mod sigv4; mod spiffe_endpoint; +mod telemetry; mod token_grant; pub mod upstream_proxy; diff --git a/crates/openshell-supervisor-network/src/policy_dns/mod.rs b/crates/openshell-supervisor-network/src/policy_dns/mod.rs index 63fb8eeca2..e15c18ba8a 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/mod.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/mod.rs @@ -35,10 +35,11 @@ pub(crate) use store::{ use crate::opa::OpaEngine; use crate::proxy::destination::{build_validation_plan, filter_resolved_addresses}; use crate::proxy::is_host_gateway_alias; +use crate::telemetry::egress_emit; use openshell_core::host_pattern::HostSelector; use openshell_ocsf::{ ActionId, ActivityId, ConfigStateChangeBuilder, DispositionId, Endpoint, - NetworkActivityBuilder, SeverityId, StateId, StatusId, ocsf_emit, + NetworkActivityBuilder, SeverityId, StateId, StatusId, }; use sha2::{Digest, Sha256}; use std::sync::Arc; @@ -489,7 +490,7 @@ fn dns_query_endpoint(name: &NormalizedName) -> Endpoint { } fn emit_dns_denial(name: &NormalizedName, detail: &str, message: &str) { - ocsf_emit!(build_dns_denial_event(name, detail, message)); + egress_emit!(build_dns_denial_event(name, detail, message)); } fn build_dns_denial_event( @@ -576,7 +577,7 @@ fn emit_dns_failure( detail: &str, message: &str, ) { - ocsf_emit!(build_dns_failure_event( + egress_emit!(build_dns_failure_event( name, family, eligible_endpoints, @@ -587,11 +588,11 @@ fn emit_dns_failure( } fn emit_mapping_publication(record: &ResolvedEndpointRecord) { - ocsf_emit!(build_mapping_publication_event(record)); + egress_emit!(build_mapping_publication_event(record)); } fn emit_observation_publication(record: &ResolvedEndpointRecord) { - ocsf_emit!(build_observation_publication_event(record)); + egress_emit!(build_observation_publication_event(record)); } fn build_observation_publication_event( diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index bd93c78c57..79b0b26f80 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -8,6 +8,8 @@ mod egress; mod relay; #[cfg(test)] pub(crate) use relay::relay_inspected_http_stream_for_test; +#[cfg(test)] +mod trace_tests; #[cfg(test)] use openshell_sandbox_backend::test_support as rpc_test; @@ -19,6 +21,7 @@ use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard}; use crate::policy_dns::PolicyEndpointId; use crate::policy_dns::{MappingLookup, MappingLookupError, ResolvedEndpointStore}; use crate::policy_local::{POLICY_LOCAL_HOST, PolicyLocalContext}; +use crate::telemetry::egress_emit; use crate::upstream_proxy::{self, UpstreamProxyConfig}; use futures::{FutureExt as _, StreamExt as _, stream::FuturesUnordered}; use http::StatusCode; @@ -44,7 +47,7 @@ use openshell_isolation_interface::contract::{ use openshell_ocsf::{ ActionId, ActivityId, BaseEventBuilder, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, HttpResponse, NetworkActivityBuilder, Process, SeverityId, StatusId, - Url as OcsfUrl, ocsf_emit, + Url as OcsfUrl, }; use openshell_sandbox_backend::NetworkMediationSource; use std::net::{IpAddr, SocketAddr}; @@ -198,14 +201,13 @@ fn build_credential_endpoint_mismatch_event( fn emit_credential_endpoint_mismatch(method: &str, host: &str, port: u16, policy_name: &str) { let event = build_credential_endpoint_mismatch_event(method, host, port, policy_name); - ocsf_emit!(event); let finding = crate::l7::build_credential_endpoint_mismatch_finding( policy_name, host, None, "Provider credential endpoint binding mismatch; request denied", ); - ocsf_emit!(finding); + crate::telemetry::emit_decision([event, finding], None, false); } /// Hostnames injected by compute drivers as `/etc/hosts` aliases for the host @@ -300,7 +302,7 @@ impl ProxyHandle { format!("Proxy listening on {local_addr}") }) .build(); - ocsf_emit!(event); + egress_emit!(event); } // Detect the trusted host gateway IP from /etc/hosts before user code @@ -341,7 +343,7 @@ impl ProxyHandle { refusing to start: {err}" )) .build(); - ocsf_emit!(event); + egress_emit!(event); miette::miette!("invalid upstream corporate proxy configuration: {err}") })?, ); @@ -355,7 +357,7 @@ impl ProxyHandle { cfg.summary() )) .build(); - ocsf_emit!(event); + egress_emit!(event); } let (exited_tx, exited_rx) = tokio::sync::oneshot::channel::<()>(); @@ -523,7 +525,7 @@ impl ProxyHandle { ) .await { - ocsf_emit!(build_proxy_connection_error_event( + egress_emit!(build_proxy_connection_error_event( peer_addr, transparent_destination_addr, format!("Proxy connection error: {err}"), @@ -532,7 +534,7 @@ impl ProxyHandle { }); } Err(ProxyAcceptError::Source(err)) => { - ocsf_emit!(build_mediation_lane_failure_event(format!( + egress_emit!(build_mediation_lane_failure_event(format!( "Network-mediation source failed; proxy accept loop exiting: {err}" ))); break; @@ -543,7 +545,7 @@ impl ProxyHandle { &mut consecutive_resource_errors, &mut consecutive_unknown_errors, ); - ocsf_emit!(build_accept_error_event(local_addr, &err, &action)); + egress_emit!(build_accept_error_event(local_addr, &err, &action)); match action { AcceptAction::Terminal => break, AcceptAction::Retry { backoff, .. } => { @@ -629,6 +631,7 @@ async fn preauthorize_transparent_open( if destination.ip() == IpAddr::V4(crate::policy_dns::POLICY_LOCAL_ADDRESS) { if destination.port() != 80 || !has_policy_local { emit_staged_transparent_denial( + None, destination, None, &binary_identity, @@ -653,6 +656,7 @@ async fn preauthorize_transparent_open( }); if let Err(denial) = identity_check { emit_staged_transparent_denial( + None, destination, None, &binary_identity, @@ -682,6 +686,7 @@ async fn preauthorize_transparent_open( Err(error) => { warn!(%destination, %error, "Denied staged transparent connection"); emit_staged_transparent_denial( + None, destination, None, &binary_identity, @@ -720,6 +725,7 @@ async fn preauthorize_transparent_open( ); warn!(%destination, %reason, "Denied staged transparent connection"); emit_staged_transparent_denial( + Some(&connect_span), destination, mapped_host, &binary_identity, @@ -772,6 +778,7 @@ async fn preauthorize_transparent_open( }; warn!(%destination, %reason, "Denied staged transparent connection"); emit_staged_transparent_denial( + Some(&connect_span), destination, mapped_host, &binary_identity, @@ -788,6 +795,7 @@ async fn preauthorize_transparent_open( { warn!(%destination, reason = %denial.reason, "Denied staged transparent destination"); emit_staged_transparent_denial( + Some(&connect_span), destination, mapped_host, &binary_identity, @@ -818,6 +826,7 @@ async fn preauthorize_transparent_open( Err(denial) => { warn!(%destination, reason = %denial.reason, "Denied staged transparent destination"); emit_staged_transparent_denial( + Some(&connect_span), destination, mapped_host, &binary_identity, @@ -844,13 +853,14 @@ async fn preauthorize_transparent_open( } fn emit_staged_transparent_denial( + parent: Option<&tracing::Span>, destination: SocketAddr, mapped_host: Option<&str>, identity: &Result, reason: &str, status_detail: &'static str, ) { - ocsf_emit!(build_staged_transparent_denial_event( + egress_emit!(optional_deny_parent: parent, build_staged_transparent_denial_event( destination, mapped_host, identity, @@ -1062,7 +1072,7 @@ impl TransparentTcpHandle { ) .await { - ocsf_emit!(build_connection_error_event( + egress_emit!(build_connection_error_event( peer_addr, format!("Transparent TCP connection error: {error}") )); @@ -1108,7 +1118,7 @@ async fn handle_transparent_tcp_connection( ) { Ok(mapping) => mapping, Err(error) => { - emit_transparent_mapping_denial(workload_addr, original, None, error); + emit_transparent_mapping_denial(None, workload_addr, original, None, error); emit_activity(&activity_tx, true, "transparent_tcp_mapping"); return Ok(()); } @@ -1129,7 +1139,7 @@ async fn handle_transparent_tcp_connection( .map_err(|error| miette::miette!("identity resolution task panicked: {error}"))?; if let NetworkAction::Deny { reason } = &decision.action { - emit_transparent_policy_denial(&decision, workload_addr, &host, port); + emit_transparent_policy_denial(&connect_span, &decision, workload_addr, &host, port); emit_denial( &denial_tx, &host, @@ -1148,6 +1158,7 @@ async fn handle_transparent_tcp_connection( if mapping.record.is_observation() { emit_transparent_mapping_denial( + Some(&connect_span), workload_addr, original, Some(&host), @@ -1166,6 +1177,7 @@ async fn handle_transparent_tcp_connection( relay::pin_policy_generation(&opa_engine, decision.policy_generation) else { emit_transparent_mapping_denial( + Some(&connect_span), workload_addr, original, Some(&host), @@ -1182,7 +1194,13 @@ async fn handle_transparent_tcp_connection( ) { Ok(mapping) => mapping, Err(error) => { - emit_transparent_mapping_denial(workload_addr, original, Some(&host), error); + emit_transparent_mapping_denial( + Some(&connect_span), + workload_addr, + original, + Some(&host), + error, + ); emit_activity(&activity_tx, true, "transparent_tcp_mapping"); return Ok(()); } @@ -1199,7 +1217,7 @@ async fn handle_transparent_tcp_connection( .find(|candidate| mapping.endpoint_ids().any(|mapped| mapped == candidate)); let Some(endpoint_id) = endpoint_id else { let reason = "authorized endpoint did not match DNS correlation"; - emit_transparent_policy_denial(&decision, workload_addr, &host, port); + emit_transparent_policy_denial(&connect_span, &decision, workload_addr, &host, port); emit_denial( &denial_tx, &host, @@ -1239,7 +1257,9 @@ async fn handle_transparent_tcp_connection( ); let middleware_gate = middleware_uninspectable_gate(&opa_engine, &ctx)?; if middleware_gate == crate::l7::middleware::UninspectableTrafficGate::Deny { - crate::l7::middleware::emit_middleware_uninspectable(&ctx, "transparent tcp", true); + connect_span.in_scope(|| { + crate::l7::middleware::emit_middleware_uninspectable(&ctx, "transparent tcp", true); + }); return Ok(()); } if middleware_gate == crate::l7::middleware::UninspectableTrafficGate::BypassWithFinding { @@ -1255,7 +1275,6 @@ async fn handle_transparent_tcp_connection( .inspect_err(|_| egress::mark_error(&dial_span)) .into_diagnostic()?; drop(dial_span); - drop(connect_span); let upstream_socket_peer = upstream.peer_addr().into_diagnostic()?; let (connected_real_destination, dial_mode) = match upstream.connect_target() { Some(upstream_proxy::ConnectTarget::Ip(ip)) => ( @@ -1267,7 +1286,7 @@ async fn handle_transparent_tcp_connection( // validated address set. A hostname-mode CONNECT would make the // corporate proxy resolve again and break that binding. Treat a // future invariant regression as an audited denial, not a panic. - emit_transparent_policy_denial(&decision, workload_addr, &host, port); + emit_transparent_policy_denial(&connect_span, &decision, workload_addr, &host, port); emit_denial( &denial_tx, &host, @@ -1298,7 +1317,8 @@ async fn handle_transparent_tcp_connection( let pid = decision .binary_pid .map_or_else(|| "-".to_string(), |pid| pid.to_string()); - ocsf_emit!(build_transparent_tcp_allow_ocsf_event( + drop(connect_span); + egress_emit!(build_transparent_tcp_allow_ocsf_event( TransparentTcpAllowAudit { workload: workload_addr, synthetic_destination: original, @@ -1428,6 +1448,7 @@ fn original_destination(stream: &TcpStream) -> std::io::Result { #[cfg(target_os = "linux")] fn emit_transparent_mapping_denial( + parent: Option<&tracing::Span>, workload: SocketAddr, original: SocketAddr, mapped_host: Option<&str>, @@ -1442,7 +1463,7 @@ fn emit_transparent_mapping_denial( | MappingLookupError::InvalidMapping | MappingLookupError::LockPoisoned => "transparent_tcp_destination_denied", }; - ocsf_emit!( + egress_emit!(optional_deny_parent: parent, NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Open) .action(ActionId::Denied) @@ -1459,6 +1480,7 @@ fn emit_transparent_mapping_denial( #[cfg(target_os = "linux")] fn emit_transparent_policy_denial( + parent: &tracing::Span, decision: &EgressDecision, workload: SocketAddr, host: &str, @@ -1476,7 +1498,7 @@ fn emit_transparent_policy_denial( let pid = decision .binary_pid .map_or_else(|| "-".to_string(), |pid| pid.to_string()); - ocsf_emit!( + egress_emit!(deny_parent: parent, NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Open) .action(ActionId::Denied) @@ -1948,6 +1970,62 @@ fn build_forward_allow_ocsf_event( .build() } +fn build_forward_rejection_event( + host: &str, + port: u16, + message: String, +) -> openshell_ocsf::OcsfEvent { + NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(host, port)) + .message(message) + .build() +} + +fn build_forward_l7_evaluation_error( + host: &str, + port: u16, + enforcement: crate::l7::EnforcementMode, + error: &str, +) -> openshell_ocsf::OcsfEvent { + let enforced = enforcement == crate::l7::EnforcementMode::Enforce; + NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .action(if enforced { + ActionId::Denied + } else { + ActionId::Other + }) + .disposition(if enforced { + DispositionId::Blocked + } else { + DispositionId::Allowed + }) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(host, port)) + .message(format!("L7 evaluation failed: {error}")) + .build() +} + +fn emit_forward_l7_decision( + decision: openshell_ocsf::OcsfEvent, + evaluation_error: Option, + connect_span: &tracing::Span, +) { + // The HTTP decision carries policy context that its evaluation-error + // companion lacks, so it must supply the shared deny span's attributes. + crate::telemetry::emit_decision( + std::iter::once(decision).chain(evaluation_error), + Some(connect_span), + false, + ); +} + fn build_forward_parse_error_ocsf_event( peer_addr: Option, method: &str, @@ -1962,6 +2040,8 @@ fn build_forward_parse_error_ocsf_event( .http_response(HttpResponse { code: StatusCode::BAD_REQUEST.as_u16(), }) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) .severity(SeverityId::Low) .status(StatusId::Failure) .message(format!("FORWARD parse error for {path}")); @@ -2190,6 +2270,7 @@ fn build_forward_destination_deny_ocsf_event( #[allow(clippy::too_many_arguments)] async fn deny_connect_destination( client: &mut C, + connect_span: &tracing::Span, denial: &DestinationDenial, peer_addr: SocketAddr, host: &str, @@ -2206,7 +2287,7 @@ where C: TokioAsyncWrite + Unpin, { let detail = destination_denial_detail(denial.kind); - ocsf_emit!(build_connect_destination_deny_ocsf_event( + egress_emit!(deny_parent: connect_span, build_connect_destination_deny_ocsf_event( denial, peer_addr, host, port, binary, pid, ancestors, cmdline, )); @@ -2239,6 +2320,7 @@ where #[allow(clippy::too_many_arguments)] async fn deny_forward_destination( client: &mut C, + connect_span: &tracing::Span, denial: &DestinationDenial, peer_addr: SocketAddr, method: &str, @@ -2258,7 +2340,7 @@ where C: TokioAsyncWrite + Unpin, { let detail = destination_denial_detail(denial.kind); - ocsf_emit!(build_forward_destination_deny_ocsf_event( + egress_emit!(deny_parent: connect_span, build_forward_destination_deny_ocsf_event( denial, peer_addr, method, host, port, path, binary, pid, ancestors, cmdline, policy, )); @@ -2717,7 +2799,7 @@ async fn handle_mediated_connection( .message(format!("CONNECT denied {host_lc}:{port}")) .status_detail(&deny_reason) .build(); - ocsf_emit!(event); + egress_emit!(deny_parent: &connect_span, event); emit_denial( &denial_tx, &host_lc, @@ -2790,6 +2872,7 @@ async fn handle_mediated_connection( } deny_connect_destination( &mut client, + &connect_span, &denial, workload_addr, &host_lc, @@ -2833,6 +2916,7 @@ async fn handle_mediated_connection( } deny_connect_destination( &mut client, + &connect_span, &denial, workload_addr, &host_lc, @@ -2882,7 +2966,7 @@ async fn handle_mediated_connection( )) .status_detail(TLS_TERMINATION_UNAVAILABLE_DETAIL) .build(); - ocsf_emit!(event); + egress_emit!(deny_parent: &connect_span, event); emit_activity_simple(activity_tx.as_ref(), true, "tls_termination_unavailable"); emit_denial( &denial_tx, @@ -2920,12 +3004,12 @@ async fn handle_mediated_connection( )) .status_detail(DETAIL) .build(); - ocsf_emit!(event); - crate::l7::emit_uninspected_credential_finding( + let finding = crate::l7::build_uninspected_credential_finding( &host_lc, policy_str, if effective_tls_skip { "tls-skip" } else { "l4" }, ); + crate::telemetry::emit_decision([event, finding], Some(&connect_span), false); emit_activity_simple(activity_tx.as_ref(), true, "uninspected_credentials"); emit_denial( &denial_tx, @@ -3002,7 +3086,7 @@ async fn handle_mediated_connection( // Log the allowed CONNECT — use CONNECT_L7 when L7 inspection follows, // so log consumers can distinguish L4-only decisions from tunnel lifecycle events. - ocsf_emit!(build_connect_allow_ocsf_event( + egress_emit!(build_connect_allow_ocsf_event( workload_addr, &host_lc, port, @@ -3113,7 +3197,7 @@ async fn handle_mediated_connection( .dst_endpoint(Endpoint::from_domain(&host_lc, port)) .message("Upstream TLS establishment failed") .build(); - ocsf_emit!(event); + egress_emit!(event); return Err(error); } }; @@ -3142,7 +3226,7 @@ async fn handle_mediated_connection( .dst_endpoint(Endpoint::from_domain(&host_lc, port)) .message(format!("TLS relay error: {e}")) .build(); - ocsf_emit!(event); + egress_emit!(event); } } } else { @@ -3174,7 +3258,7 @@ async fn handle_mediated_connection( .message(format!("CONNECT refused for {host_lc}:{port}: {DETAIL}")) .status_detail(DETAIL) .build(); - ocsf_emit!(event); + egress_emit!(event); emit_activity_simple(activity_tx.as_ref(), true, "tls_termination_unavailable"); emit_denial( &denial_tx, @@ -3217,7 +3301,7 @@ async fn handle_mediated_connection( .dst_endpoint(Endpoint::from_domain(&host_lc, port)) .message(message) .build(); - ocsf_emit!(event); + egress_emit!(event); } } } else { @@ -3247,7 +3331,7 @@ async fn handle_mediated_connection( )) .status_detail(protocol_detail) .build(); - ocsf_emit!(event); + egress_emit!(event); emit_activity_simple(activity_tx.as_ref(), true, "l7_parse_rejection"); emit_denial( &denial_tx, @@ -3848,7 +3932,7 @@ fn emit_l7_tunnel_close_after_policy_change(host: &str, port: u16, error: miette "L7 tunnel closed before inspection because policy changed: {error}" )) .build(); - ocsf_emit!(event); + egress_emit!(event); } async fn reject_stale_connect_policy( @@ -4690,7 +4774,7 @@ fn parse_allowed_ips(raw: &[String]) -> std::result::Result, S n.prefix_len() )) .build(); - ocsf_emit!(event); + egress_emit!(event); } nets.push(n); } @@ -5364,7 +5448,7 @@ async fn handle_forward_proxy( // canonicalized below before credential binding, policy-path evaluation, // upstream bytes, or telemetry consume it. let Ok((scheme, host, port, mut path)) = parse_proxy_uri(target_uri) else { - ocsf_emit!(build_forward_parse_error_ocsf_event( + egress_emit!(build_forward_parse_error_ocsf_event( workload_peer_addr, method, &telemetry_path @@ -5433,7 +5517,7 @@ async fn handle_forward_proxy( if scheme != "http" { let event = build_forward_unsupported_scheme_ocsf_event(method, &scheme, &host_lc, port); - ocsf_emit!(event); + egress_emit!(event); if scheme == "https" { respond( client, @@ -5525,7 +5609,7 @@ async fn handle_forward_proxy( let matched_policy = match &decision.action { NetworkAction::Allow { matched_policy } => matched_policy.clone(), NetworkAction::Deny { reason } => { - ocsf_emit!(build_forward_policy_deny_ocsf_event( + egress_emit!(deny_parent: &connect_span, build_forward_policy_deny_ocsf_event( workload_addr, method, &host_lc, @@ -5636,16 +5720,12 @@ async fn handle_forward_proxy( let prepared_target = match prepare_forward_target(&path, canonicalize_options) { Ok(prepared) => prepared, Err(error) => { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .message(format!( - "FORWARD rejecting non-canonical request-target: {error}" - )) - .build(); - ocsf_emit!(event); + let event = build_forward_rejection_event( + &host_lc, + port, + format!("FORWARD rejecting non-canonical request-target: {error}"), + ); + egress_emit!(deny_parent: &connect_span, event); emit_activity_simple(activity_tx, true, "forward_parse_rejection"); respond( client, @@ -5798,7 +5878,7 @@ async fn handle_forward_proxy( if !l7_config.config.allow_encoded_slash && crate::l7::path::canonical_path_has_encoded_slash(&path) { - ocsf_emit!(build_forward_l7_parse_rejection_ocsf_event( + egress_emit!(deny_parent: &connect_span, build_forward_l7_parse_rejection_ocsf_event( workload_addr, method, &host_lc, @@ -5855,7 +5935,7 @@ async fn handle_forward_proxy( )) .status_detail(upgrade_detail) .build(); - ocsf_emit!(event); + egress_emit!(deny_parent: &connect_span, event); emit_activity_simple(activity_tx, true, "l7_parse_rejection"); emit_denial_simple( denial_tx, @@ -5913,14 +5993,12 @@ async fn handle_forward_proxy( { Ok(info) => info, Err(e) => { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .message(format!("FORWARD_GRAPHQL_L7 request rejected: {e}")) - .build(); - ocsf_emit!(event); + let event = build_forward_rejection_event( + &host_lc, + port, + format!("FORWARD_GRAPHQL_L7 request rejected: {e}"), + ); + egress_emit!(deny_parent: &connect_span, event); emit_activity_simple(activity_tx, true, "l7_parse_rejection"); respond( client, @@ -5967,14 +6045,12 @@ async fn handle_forward_proxy( { Ok(body) => body, Err(e) => { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .message(format!("FORWARD_JSONRPC_L7 request rejected: {e}")) - .build(); - ocsf_emit!(event); + let event = build_forward_rejection_event( + &host_lc, + port, + format!("FORWARD_JSONRPC_L7 request rejected: {e}"), + ); + egress_emit!(deny_parent: &connect_span, event); emit_activity_simple(activity_tx, true, "l7_parse_rejection"); respond( client, @@ -6025,18 +6101,17 @@ async fn handle_forward_proxy( let hard_deny_reason = crate::l7::relay::l7_request_hard_deny_reason(l7_config.config.protocol, &request_info); let force_deny = hard_deny_reason.is_some(); + let mut evaluation_error = None; let (allowed, reason) = hard_deny_reason.map_or_else( || { crate::l7::relay::evaluate_l7_request(&tunnel_engine, &l7_ctx, &request_info) .unwrap_or_else(|e| { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Low) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .message(format!("L7 eval failed, denying request: {e}")) - .build(); - ocsf_emit!(event); + evaluation_error = Some(build_forward_l7_evaluation_error( + &host_lc, + port, + l7_config.config.enforcement, + &e.to_string(), + )); (false, format!("L7 evaluation error: {e}")) }) }, @@ -6112,7 +6187,7 @@ async fn handle_forward_proxy( .firewall_rule(policy_str, engine_type) .message(log_message) .build(); - ocsf_emit!(event); + emit_forward_l7_decision(event, evaluation_error, &connect_span); } let effectively_denied = force_deny @@ -6163,6 +6238,7 @@ async fn handle_forward_proxy( Err(denial) => { deny_forward_destination( client, + &connect_span, &denial, workload_addr, method, @@ -6201,6 +6277,7 @@ async fn handle_forward_proxy( Err(denial) => { deny_forward_destination( client, + &connect_span, &denial, workload_addr, method, @@ -6621,7 +6698,7 @@ async fn handle_forward_proxy( "FORWARD upstream connect failed for {host_lc}:{port}: {e}" )) .build(); - ocsf_emit!(event); + egress_emit!(deny_parent: &connect_span, event); if let Some(session) = middleware_session.take() { session .end(openshell_core::proto::MiddlewareSessionEndReason::UpstreamFailure) @@ -6751,7 +6828,7 @@ async fn handle_forward_proxy( // The request has now survived middleware, token grant, credential // rewriting, generation checks, and the HTTP relay. Only now record the // final allowed outcome. - ocsf_emit!(build_forward_allow_ocsf_event( + egress_emit!(build_forward_allow_ocsf_event( workload_addr, method, &host_lc, diff --git a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs index 010cb466c5..95598fb8c2 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs @@ -89,6 +89,7 @@ async fn destination_denials_preserve_adapter_specific_wire_contracts() { let (mut app, mut proxy) = tcp_pair().await; deny_connect_destination( &mut proxy, + &tracing::Span::none(), &denial, peer, "target.example", @@ -116,6 +117,7 @@ async fn destination_denials_preserve_adapter_specific_wire_contracts() { let (mut app, mut proxy) = tcp_pair().await; deny_forward_destination( &mut proxy, + &tracing::Span::none(), &denial, peer, "POST", diff --git a/crates/openshell-supervisor-network/src/proxy/trace_tests.rs b/crates/openshell-supervisor-network/src/proxy/trace_tests.rs new file mode 100644 index 0000000000..4390ec53f1 --- /dev/null +++ b/crates/openshell-supervisor-network/src/proxy/trace_tests.rs @@ -0,0 +1,433 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::telemetry::tests::{assert_deny_resolves, exported}; +use opentelemetry_proto::tonic::common::v1::any_value::Value; + +fn emit_staged_policy_deny(debug: bool) { + let intent = EgressIntent::connect("blocked.example".to_string(), 443); + let connect = egress::connect_span(&intent); + assert_eq!(connect.is_disabled(), !debug); + connect.in_scope(|| { + egress::traced_authorization( + intent, + |intent| EgressDecision { + intent, + action: NetworkAction::Deny { + reason: "not permitted".to_string(), + }, + policy_generation: 1, + identity: ProcessIdentityEvidence::Unavailable( + IdentityUnavailableReason::LookupFailed, + ), + endpoint: EndpointDecision::default(), + binary: None, + binary_pid: None, + ancestors: vec![], + cmdline_paths: vec![], + }, + |decision| decision, + ); + }); + // Authorization has exited. The actual staged L4 emission must enter its + // own INFO deny span, retaining the saved DEBUG parent when it is enabled. + emit_staged_transparent_denial( + Some(&connect), + "192.0.2.1:443".parse().unwrap(), + Some("blocked.example"), + &Err(ResolveError::Failed("unavailable".to_string())), + "not permitted", + "transparent_tcp_policy_denied", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn default_level_l4_deny_uid_resolves_to_one_span_root_trace() { + let (events, received) = exported("warn", || emit_staged_policy_deny(false)).await; + assert_eq!(events.len(), 1); + assert_deny_resolves(&events, &received, true); + assert_eq!(received.spans.len(), 1); + assert_eq!(events[0]["class_uid"], 4001); + let attributes = received.spans[0] + .attributes + .iter() + .map(|kv| { + ( + kv.key.as_str(), + kv.value.as_ref().unwrap().value.as_ref().unwrap(), + ) + }) + .collect::>(); + assert_eq!( + attributes["openshell.policy.decision"], + &Value::StringValue("deny".to_string()) + ); + assert_eq!( + attributes["server.address"], + &Value::StringValue("blocked.example".to_string()) + ); + assert_eq!(attributes["server.port"], &Value::IntValue(443)); + assert_eq!( + attributes["openshell.policy.reason"], + &Value::StringValue("transparent_tcp_policy_denied".to_string()) + ); + assert!(!attributes.contains_key("openshell.policy.name")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn debug_level_deny_retains_connect_and_authorization_trace() { + let (events, received) = exported("debug", || emit_staged_policy_deny(true)).await; + assert_eq!(events.len(), 1); + assert_eq!( + received + .spans + .iter() + .filter(|span| span.name == "supervisor.egress.deny") + .count(), + 1 + ); + assert_deny_resolves(&events, &received, false); + let connect = received + .spans + .iter() + .find(|span| span.name == "supervisor.egress.connect") + .unwrap(); + let deny = received + .spans + .iter() + .find(|span| span.name == "supervisor.egress.deny") + .unwrap(); + let authorize = received + .spans + .iter() + .find(|span| span.name == "supervisor.egress.authorize") + .unwrap(); + assert_eq!(deny.trace_id, connect.trace_id); + assert_eq!(deny.parent_span_id, connect.span_id); + assert_eq!(authorize.parent_span_id, connect.span_id); + assert_eq!(authorize.trace_id, connect.trace_id); + assert_eq!(events[0]["trace"]["uid"], hex::encode(&connect.trace_id)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn paired_credential_denial_shares_one_deny_span() { + let (events, received) = exported("warn", || { + emit_credential_endpoint_mismatch("POST", "blocked.example", 443, "credential-policy"); + }) + .await; + assert_eq!(events.len(), 2); + assert_eq!(received.spans.len(), 1); + assert_eq!(events[0]["trace"]["uid"], events[1]["trace"]["uid"]); + assert_deny_resolves(&events, &received, true); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn repeated_credential_denials_keep_decision_spans_separate() { + for level in ["warn", "debug"] { + let (events, received) = exported(level, || { + let connect = + egress::connect_span(&EgressIntent::connect("blocked.example".to_string(), 443)); + connect.in_scope(|| { + for _ in 0..2 { + emit_credential_endpoint_mismatch( + "POST", + "blocked.example", + 443, + "credential-policy", + ); + } + }); + let allowed = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .action(ActionId::Allowed) + .dst_endpoint(Endpoint::from_domain("allowed.example", 443)) + .build(); + crate::telemetry::emit(allowed, None, false); + }) + .await; + + assert_eq!(events.len(), 5); + assert_deny_resolves(&events[..4], &received, level == "warn"); + for pair in events[..4].chunks_exact(2) { + assert_eq!(pair[0]["trace"]["uid"], pair[1]["trace"]["uid"]); + } + assert!(events[4].get("trace").is_none(), "deny context must exit"); + let denies = received + .spans + .iter() + .filter(|span| span.name == "supervisor.egress.deny") + .collect::>(); + assert_eq!(denies.len(), 2); + assert_ne!(denies[0].span_id, denies[1].span_id); + assert_eq!(received.spans.len(), if level == "debug" { 3 } else { 2 }); + if level == "debug" { + let connect = received + .spans + .iter() + .find(|span| span.name == "supervisor.egress.connect") + .unwrap(); + for deny in denies { + assert_eq!(deny.trace_id, connect.trace_id); + assert_eq!(deny.parent_span_id, connect.span_id); + } + assert_eq!(events[0]["trace"]["uid"], events[2]["trace"]["uid"]); + } else { + assert_ne!(events[0]["trace"]["uid"], events[2]["trace"]["uid"]); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn noncanonical_forward_rejections_have_a_deny_trace() { + let (events, received) = exported("warn", || { + let Err(error) = + prepare_forward_target("/%zz", crate::l7::path::CanonicalizeOptions::default()) + else { + panic!("invalid escape must be rejected"); + }; + egress_emit!(build_forward_rejection_event( + "blocked.example", + 443, + format!("FORWARD rejecting non-canonical request-target: {error}") + )); + }) + .await; + assert_eq!(events.len(), 1); + assert_eq!(events[0]["action_id"], 2); + assert_deny_resolves(&events, &received, true); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn connect_and_forward_denials_use_the_saved_connect_parent() { + for level in ["warn", "debug"] { + for forward in [false, true] { + let (events, received) = exported(level, || { + let intent = EgressIntent::connect("blocked.example".to_string(), 443); + let connect = egress::connect_span(&intent); + let decision = EgressDecision { + intent, + action: NetworkAction::Allow { + matched_policy: None, + }, + policy_generation: 1, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::default(), + binary: None, + binary_pid: None, + ancestors: vec![], + cmdline_paths: vec![], + }; + let denial = DestinationDenial { + kind: DestinationDenialKind::InternalAddress, + reason: "private destination".to_string(), + }; + let mut output = Vec::new(); + if forward { + futures::executor::block_on(deny_forward_destination( + &mut output, + &connect, + &denial, + "127.0.0.1:41000".parse().unwrap(), + "POST", + "blocked.example", + 443, + "/", + "curl", + "42", + "-", + "curl", + "forward-policy", + &decision, + None, + None, + )) + .unwrap(); + } else { + futures::executor::block_on(deny_connect_destination( + &mut output, + &connect, + &denial, + "127.0.0.1:41000".parse().unwrap(), + "blocked.example", + 443, + "curl", + "42", + "-", + "curl", + &decision, + &None, + &None, + )) + .unwrap(); + } + assert!(output.starts_with(b"HTTP/1.1 403 Forbidden")); + }) + .await; + assert_eq!(events.len(), 1); + assert_deny_resolves(&events, &received, level == "warn"); + assert_eq!( + received + .spans + .iter() + .filter(|span| span.name == "supervisor.egress.deny") + .count(), + 1 + ); + if level == "debug" { + let connect = received + .spans + .iter() + .find(|span| span.name == "supervisor.egress.connect") + .unwrap(); + let deny = received + .spans + .iter() + .find(|span| span.name == "supervisor.egress.deny") + .unwrap(); + assert_eq!(deny.parent_span_id, connect.span_id); + assert_eq!(events[0]["trace"]["uid"], hex::encode(&connect.trace_id)); + } + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn evaluation_errors_only_trace_enforced_denials() { + for mode in [ + crate::l7::EnforcementMode::Audit, + crate::l7::EnforcementMode::Enforce, + ] { + let (events, received) = exported("warn", || { + egress_emit!(build_forward_l7_evaluation_error( + "blocked.example", + 443, + mode, + "evaluation unavailable" + )); + tracing::info_span!("test.export-barrier").in_scope(|| {}); + }) + .await; + assert_eq!(events.len(), 1); + if mode == crate::l7::EnforcementMode::Enforce { + assert_eq!(events[0]["action_id"], 2); + assert_deny_resolves(&events, &received, true); + } else { + assert!(events[0].get("trace").is_none()); + assert_eq!( + received + .spans + .iter() + .filter(|span| span.name == "supervisor.egress.deny") + .count(), + 0 + ); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn forward_evaluation_error_keeps_the_http_decision_policy_context() { + for level in ["warn", "debug"] { + for mode in [ + crate::l7::EnforcementMode::Audit, + crate::l7::EnforcementMode::Enforce, + ] { + let enforced = mode == crate::l7::EnforcementMode::Enforce; + let (events, received) = exported(level, || { + let connect = egress::connect_span(&EgressIntent::forward_http( + "blocked.example".to_string(), + 443, + )); + let decision = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .action(if enforced { + ActionId::Denied + } else { + ActionId::Allowed + }) + .disposition(if enforced { + DispositionId::Blocked + } else { + DispositionId::Allowed + }) + .http_request(HttpRequest::new( + "GET", + OcsfUrl::new("http", "blocked.example", "/", 443), + )) + .dst_endpoint(Endpoint::from_domain("blocked.example", 443)) + .firewall_rule("forward-policy", "l7") + .build(); + let error = build_forward_l7_evaluation_error( + "blocked.example", + 443, + mode, + "evaluation unavailable", + ); + // Use the same grouping as the forward-proxy production path. + emit_forward_l7_decision(decision, Some(error), &connect); + if !enforced { + tracing::info_span!("test.export-barrier").in_scope(|| {}); + } + }) + .await; + + assert_eq!(events.len(), 2); + let http_decision = events + .iter() + .find(|event| event["class_uid"] == 4002) + .unwrap(); + let evaluation_error = events + .iter() + .find(|event| event["class_uid"] == 4001) + .unwrap(); + let denies = received + .spans + .iter() + .filter(|span| span.name == "supervisor.egress.deny") + .collect::>(); + if enforced { + assert_eq!(denies.len(), 1); + assert_eq!(received.spans.len(), if level == "debug" { 2 } else { 1 }); + assert_eq!(http_decision["action_id"], 2); + assert_eq!(evaluation_error["action_id"], 2); + assert_eq!(events[0]["trace"]["uid"], events[1]["trace"]["uid"]); + assert_deny_resolves(&events, &received, level == "warn"); + let attribute = |key: &str| { + denies[0] + .attributes + .iter() + .find(|attribute| attribute.key == key) + .and_then(|attribute| attribute.value.as_ref()?.value.as_ref()) + }; + assert_eq!( + attribute("openshell.policy.name"), + Some(&Value::StringValue("forward-policy".to_string())), + "the shared deny span must retain the HTTP decision's policy name" + ); + assert_eq!( + attribute("server.address"), + Some(&Value::StringValue("blocked.example".to_string())) + ); + assert_eq!(attribute("ocsf.class_uid"), Some(&Value::IntValue(4002))); + // This production decision currently has no status_detail. + assert!(attribute("openshell.policy.reason").is_none()); + if level == "debug" { + let connect = received + .spans + .iter() + .find(|span| span.name == "supervisor.egress.connect") + .unwrap(); + assert_eq!(denies[0].parent_span_id, connect.span_id); + assert_eq!(denies[0].trace_id, connect.trace_id); + } + } else { + assert!(denies.is_empty()); + assert_eq!(http_decision["action_id"], 1); + assert_eq!(evaluation_error["action_id"], 99); + for event in events { + assert!(event.get("trace").is_none()); + } + } + } + } +} diff --git a/crates/openshell-supervisor-network/src/telemetry.rs b/crates/openshell-supervisor-network/src/telemetry.rs new file mode 100644 index 0000000000..02fb137133 --- /dev/null +++ b/crates/openshell-supervisor-network/src/telemetry.rs @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Keep a deny span active during OCSF emission. + +use openshell_ocsf::{ActionId, OcsfEvent}; + +/// Emit egress events inside an INFO deny span. `denied:` supplies the decision +/// for findings without an action; domain events carry their own action. +/// A saved connect span parents denies after authorization exits. +macro_rules! egress_emit { + ($event:expr) => { + $crate::telemetry::emit($event, None, false) + }; + (deny_parent: $parent:expr, $event:expr) => { + $crate::telemetry::emit($event, Some($parent), false) + }; + (deny_parent: $parent:expr, denied: $denied:expr, $event:expr) => {{ + // Evaluate the decision before moving the event it may inspect. + let denied = $denied; + $crate::telemetry::emit($event, Some($parent), denied) + }}; + (optional_deny_parent: $parent:expr, $event:expr) => { + $crate::telemetry::emit($event, $parent, false) + }; + (denied: $denied:expr, $event:expr) => {{ + // Evaluate the decision before moving the event it may inspect. + let denied = $denied; + $crate::telemetry::emit($event, None, denied) + }}; +} +pub(crate) use egress_emit; + +fn is_denied(event: &OcsfEvent) -> bool { + match event { + OcsfEvent::NetworkActivity(event) => event.action == Some(ActionId::Denied), + OcsfEvent::HttpActivity(event) => event.action == Some(ActionId::Denied), + OcsfEvent::DetectionFinding(event) => event.action == Some(ActionId::Denied), + _ => false, + } +} + +pub fn emit(event: OcsfEvent, parent: Option<&tracing::Span>, denied: bool) { + if !denied && !is_denied(&event) { + openshell_ocsf::ocsf_emit!(event); + return; + } + let span = deny_span(&event, parent); + span.in_scope(|| openshell_ocsf::ocsf_emit!(event)); +} + +/// Emit all records of one synchronous decision under one deny span. Different +/// decisions remain separate, even when they concern the same connection. +pub fn emit_decision( + events: impl IntoIterator, + parent: Option<&tracing::Span>, + denied: bool, +) { + let events: Vec<_> = events.into_iter().collect(); + let primary = events + .iter() + .find(|event| is_denied(event)) + .or_else(|| if denied { events.first() } else { None }); + let span = primary.map(|event| deny_span(event, parent)); + let emit = || { + for event in events { + openshell_ocsf::ocsf_emit!(event); + } + }; + if let Some(span) = span { + span.in_scope(emit); + } else { + emit(); + } +} + +/// Build a decision scope, reusing an active deny scope for its companion +/// records rather than creating another exported span. +pub fn deny_span(event: &OcsfEvent, parent: Option<&tracing::Span>) -> tracing::Span { + let current = tracing::Span::current(); + if current + .metadata() + .is_some_and(|metadata| metadata.name() == "supervisor.egress.deny") + { + return current; + } + let (endpoint, policy) = match event { + OcsfEvent::NetworkActivity(e) => (e.dst_endpoint.as_ref(), e.firewall_rule.as_ref()), + OcsfEvent::HttpActivity(e) => (e.dst_endpoint.as_ref(), e.firewall_rule.as_ref()), + _ => (None, None), + }; + + // Finding evidence supplies decision fields, not a trace parent. + let evidence = |key: &str| match event { + OcsfEvent::DetectionFinding(e) => e.evidences.as_ref().and_then(|items| { + items + .iter() + .find_map(|item| item.data.as_ref()?.get(key)?.as_str()) + }), + _ => None, + }; + let policy = policy + .map(|rule| rule.name.as_str()) + .or_else(|| evidence("policy")); + let address = endpoint + .and_then(|endpoint| endpoint.domain.as_deref().or(endpoint.ip.as_deref())) + .or_else(|| evidence("host")); + let port = endpoint.and_then(|endpoint| endpoint.port); + let reason = event.base().status_detail.as_deref(); + let parent = parent.cloned().unwrap_or_else(tracing::Span::current); + let span = tracing::info_span!( + parent: &parent, + "supervisor.egress.deny", + openshell.policy.decision = "deny", + openshell.policy.name = tracing::field::Empty, + openshell.policy.reason = tracing::field::Empty, + server.address = tracing::field::Empty, + server.port = tracing::field::Empty, + ocsf.class_uid = i64::from(event.class_uid()), + ); + if let Some(policy) = policy { + span.record("openshell.policy.name", policy); + } + if let Some(reason) = reason { + span.record("openshell.policy.reason", reason); + } + if let Some(address) = address { + span.record("server.address", address); + } + if let Some(port) = port { + span.record("server.port", i64::from(port)); + } + span +} + +#[cfg(test)] +pub mod tests; diff --git a/crates/openshell-supervisor-network/src/telemetry/tests.rs b/crates/openshell-supervisor-network/src/telemetry/tests.rs new file mode 100644 index 0000000000..b2940113ab --- /dev/null +++ b/crates/openshell-supervisor-network/src/telemetry/tests.rs @@ -0,0 +1,362 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::{Arc, LazyLock, Mutex}; + +use openshell_otel_test_support::{OtlpTestServer, ReceivedTraces, tracing_test_lock}; +use tracing_subscriber::prelude::*; + +#[derive(Clone)] +struct Writer(Arc>>); +impl std::io::Write for Writer { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// Capture the actual OCSF dispatch and receive spans over OTLP/gRPC. +pub async fn exported( + log_level: &str, + emit: impl FnOnce(), +) -> (Vec, ReceivedTraces) { + static REGISTER: LazyLock<()> = LazyLock::new(|| { + assert!(openshell_ocsf::register_trace_correlation_extractor( + openshell_otel::current_ocsf_trace_correlation + )); + }); + let _lock = tracing_test_lock().await; + LazyLock::force(®ISTER); + let collector = OtlpTestServer::start().await; + let (provider, error) = openshell_otel::provider_for(Some(openshell_otel::OtlpExportConfig { + endpoint: collector.endpoint(), + service_name: openshell_otel::ServiceName::Fixed("openshell-supervisor"), + service_version: None, + resource_attributes: vec![], + })); + assert!(error.is_none()); + let provider = provider.unwrap(); + let bytes = Arc::new(Mutex::new(vec![])); + let subscriber = tracing_subscriber::registry() + .with( + openshell_ocsf::OcsfJsonlLayer::new(Writer(bytes.clone())) + .with_filter(tracing_subscriber::filter::LevelFilter::INFO), + ) + .with( + openshell_otel::layer(&provider, "openshell-supervisor") + .with_filter(openshell_otel::supervisor_span_filter(log_level)), + ) + // The shared test registry keeps callsites enabled process-wide. + // Gate this subscriber too, so the default case also checks a disabled + // DEBUG connect handle, as well as absence from the OTLP export. + .with(openshell_otel::supervisor_span_filter(log_level)); + tracing::subscriber::with_default(subscriber, emit); + provider.force_flush().unwrap(); + collector.wait_for_export().await; + provider.shutdown().unwrap(); + let received = collector.shutdown().await; + let events = String::from_utf8(bytes.lock().unwrap().clone()) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + (events, received) +} + +pub fn assert_deny_resolves(events: &[serde_json::Value], received: &ReceivedTraces, root: bool) { + use opentelemetry_proto::tonic::common::v1::any_value::Value; + + assert!(!events.is_empty()); + for event in events { + let uid = event["trace"]["uid"] + .as_str() + .expect("emission was inside sampled deny span"); + assert_eq!(uid.len(), 32); + assert!( + event["metadata"]["profiles"] + .as_array() + .unwrap() + .iter() + .any(|p| p == "trace") + ); + let span = received + .spans + .iter() + .find(|span| { + span.name == "supervisor.egress.deny" && hex::encode(&span.trace_id) == uid + }) + .expect("OCSF trace.uid resolves to exported deny span"); + if root { + assert!(span.parent_span_id.is_empty()); + assert_eq!( + received + .spans + .iter() + .filter(|other| other.trace_id == span.trace_id) + .count(), + 1 + ); + } + let attributes = span + .attributes + .iter() + .map(|kv| kv.key.as_str()) + .collect::>(); + for key in ["openshell.policy.decision", "ocsf.class_uid"] { + assert!(attributes.contains(&key), "deny span carries {key}"); + } + let class_uid = span + .attributes + .iter() + .find(|attribute| attribute.key == "ocsf.class_uid") + .and_then(|attribute| attribute.value.as_ref()?.value.as_ref()); + assert!( + matches!(class_uid, Some(Value::IntValue(_))), + "deny span exports ocsf.class_uid as an integer: {class_uid:?}" + ); + } +} + +pub fn assert_deny_attribute(received: &ReceivedTraces, key: &str, expected: &str) { + use opentelemetry_proto::tonic::common::v1::any_value::Value; + let deny = received + .spans + .iter() + .find(|span| span.name == "supervisor.egress.deny") + .unwrap(); + let attribute = deny + .attributes + .iter() + .find(|attribute| attribute.key == key) + .unwrap(); + assert_eq!( + attribute.value.as_ref().unwrap().value.as_ref().unwrap(), + &Value::StringValue(expected.to_string()) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn companion_emissions_reuse_an_active_decision_scope() { + let (events, received) = exported("warn", || { + let event = openshell_ocsf::NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .action(openshell_ocsf::ActionId::Denied) + .dst_endpoint(openshell_ocsf::Endpoint::from_domain( + "blocked.example", + 443, + )) + .build(); + let finding = + openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()).build(); + let scope = super::deny_span(&event, None); + scope.in_scope(|| { + super::emit(event, None, false); + super::emit(finding, None, true); + }); + }) + .await; + assert_eq!(events.len(), 2); + assert_eq!(received.spans.len(), 1); + assert_eq!(events[0]["trace"]["uid"], events[1]["trace"]["uid"]); + assert_deny_resolves(&events, &received, true); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn concurrent_decisions_keep_companions_and_request_contexts_separate() { + for level in ["warn", "debug"] { + let (events, received) = exported(level, || { + let dispatch = tracing::dispatcher::get_default(Clone::clone); + let barrier = std::sync::Barrier::new(2); + std::thread::scope(|threads| { + for host in ["alpha.example", "beta.example"] { + let dispatch = &dispatch; + let barrier = &barrier; + threads.spawn(move || { + tracing::dispatcher::with_default(dispatch, || { + let outside = openshell_ocsf::ctx::ctx().sandbox_id.clone(); + let mut context = (*openshell_ocsf::ctx::ctx()).clone(); + context.sandbox_id = host.to_string(); + context.sandbox_name = format!("sandbox-{host}"); + openshell_ocsf::ctx::bind_thread_ctx(Some(Arc::new(context))); + let connect = tracing::debug_span!("supervisor.egress.connect"); + connect.in_scope(|| { + for attempt in 0..2 { + let decision = format!("{host}:{attempt}"); + let event = openshell_ocsf::NetworkActivityBuilder::new( + openshell_ocsf::ctx::ctx(), + ) + .action(openshell_ocsf::ActionId::Denied) + .dst_endpoint(openshell_ocsf::Endpoint::from_domain(host, 443)) + .status_detail(&decision) + .message(&decision) + .build(); + let finding = openshell_ocsf::DetectionFindingBuilder::new( + openshell_ocsf::ctx::ctx(), + ) + .message(&decision) + .build(); + let scope = super::deny_span(&event, None); + scope.in_scope(|| { + super::emit(event, None, false); + // Both threads keep different deny scopes entered + // while their companion records interleave. + barrier.wait(); + super::emit(finding, None, true); + barrier.wait(); + }); + } + }); + openshell_ocsf::ctx::bind_thread_ctx(None); + assert_eq!(openshell_ocsf::ctx::ctx().sandbox_id, outside); + }); + }); + } + }); + }) + .await; + + assert_eq!(events.len(), 8); + assert_deny_resolves(&events, &received, level == "warn"); + let denies = received + .spans + .iter() + .filter(|span| span.name == "supervisor.egress.deny") + .collect::>(); + assert_eq!(denies.len(), 4); + assert_eq!(received.spans.len(), if level == "debug" { 6 } else { 4 }); + let attribute = |span: &opentelemetry_proto::tonic::trace::v1::Span, key: &str| { + use opentelemetry_proto::tonic::common::v1::any_value::Value; + match span + .attributes + .iter() + .find(|attribute| attribute.key == key) + .and_then(|attribute| attribute.value.as_ref()) + .and_then(|value| value.value.as_ref()) + { + Some(Value::StringValue(value)) => value.clone(), + other => panic!("expected string attribute {key}, got {other:?}"), + } + }; + let mut traces_by_host = Vec::new(); + for host in ["alpha.example", "beta.example"] { + let mut traces = Vec::new(); + for attempt in 0..2 { + let decision = format!("{host}:{attempt}"); + let pair = events + .iter() + .filter(|event| event["message"] == decision) + .collect::>(); + assert_eq!(pair.len(), 2); + for event in &pair { + assert_eq!(event["container"]["uid"], host); + assert_eq!(event["container"]["name"], format!("sandbox-{host}")); + } + assert_eq!(pair[0]["trace"]["uid"], pair[1]["trace"]["uid"]); + let spans = denies + .iter() + .filter(|span| attribute(span, "openshell.policy.reason") == decision) + .collect::>(); + assert_eq!(spans.len(), 1); + let deny = spans[0]; + assert_eq!(attribute(deny, "server.address"), host); + assert_eq!(pair[0]["trace"]["uid"], hex::encode(&deny.trace_id)); + if level == "debug" { + let connect = received + .spans + .iter() + .find(|span| span.span_id == deny.parent_span_id) + .unwrap(); + assert_eq!(connect.name, "supervisor.egress.connect"); + assert_eq!(connect.trace_id, deny.trace_id); + } + traces.push(deny.trace_id.clone()); + } + if level == "debug" { + assert_eq!(traces[0], traces[1]); + } else { + assert_ne!(traces[0], traces[1]); + } + traces_by_host.push(traces); + } + for alpha in &traces_by_host[0] { + for beta in &traces_by_host[1] { + assert_ne!( + alpha, beta, + "independent request contexts must stay separate" + ); + } + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn missing_deny_context_leaves_optional_attributes_unset() { + let (events, received) = exported("warn", || { + let finding = openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .action(openshell_ocsf::ActionId::Denied) + .message("diagnostic text is not a policy reason") + .build(); + super::emit(finding, None, false); + let network = openshell_ocsf::NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .action(openshell_ocsf::ActionId::Denied) + .dst_endpoint(openshell_ocsf::Endpoint { + name: None, + domain: None, + ip: None, + port: None, + }) + .build(); + super::emit(network, None, false); + }) + .await; + assert_deny_resolves(&events, &received, true); + assert_eq!(events.len(), 2); + assert_eq!(received.spans.len(), 2); + for span in &received.spans { + for key in [ + "openshell.policy.name", + "openshell.policy.reason", + "server.address", + "server.port", + ] { + assert!( + span.attributes.iter().all(|kv| kv.key != key), + "{key} stays unset" + ); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn allowed_emissions_do_not_create_info_deny_spans() { + let (events, received) = exported("warn", || { + let allowed = openshell_ocsf::NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .action(openshell_ocsf::ActionId::Allowed) + .dst_endpoint(openshell_ocsf::Endpoint::from_domain( + "allowed.example", + 443, + )) + .build(); + super::emit(allowed, None, false); + crate::l7::middleware::emit_middleware_uninspectable( + &crate::l7::relay::L7EvalContext::default(), + "unknown protocol", + false, + ); + // Flush an unrelated INFO span so the collector wait is bounded by an export. + tracing::info_span!("test.export-barrier").in_scope(|| {}); + }) + .await; + assert_eq!(events.len(), 2); + for event in events { + assert!(event.get("trace").is_none()); + } + assert!( + !received + .spans + .iter() + .any(|span| span.name == "supervisor.egress.deny") + ); +} diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index de29e9509a..8ab8d02a52 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -14,7 +14,6 @@ use miette::{IntoDiagnostic, Result}; use openshell_isolation_interface::contract::BackendDescriptor; use tracing::{error, info, warn}; use tracing_subscriber::EnvFilter; -use tracing_subscriber::filter::LevelFilter; use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::SubscriberInitExt as _}; mod logging; @@ -402,6 +401,10 @@ fn main() -> Result<()> { return openshell_supervisor::check_control_readiness(&args.socket); } + openshell_ocsf::register_trace_correlation_extractor( + openshell_otel::current_ocsf_trace_correlation, + ); + let args = Args::parse(); if args.mode == SupervisorMode::Shared { return shared::run(args); @@ -509,7 +512,7 @@ fn main() -> Result<()> { otlp_layer_provider .as_ref() .map(|provider| openshell_otel::layer(provider, SERVICE_NAME)) - .with_filter(otlp_span_filter(&args.log_level)), + .with_filter(openshell_otel::supervisor_span_filter(&args.log_level)), ) .init(); if !file_available { @@ -694,16 +697,6 @@ fn supervisor_resource_attributes( .collect() } -/// A more verbose log level raises `openshell*` spans; dependency spans stay at -/// INFO so they do not flood the exporter queue. -fn otlp_span_filter(log_level: &str) -> EnvFilter { - let level = log_level - .parse::() - .unwrap_or(LevelFilter::INFO) - .max(LevelFilter::INFO); - EnvFilter::new(format!("info,openshell={level}")) -} - /// Choose the supervisor's exit status for a sandbox run. /// /// The reserved status tells the compute driver that the sandbox rejected the @@ -808,7 +801,8 @@ mod tests { #[test] fn otlp_span_filter_raises_only_openshell_targets() { - let directives = |log_level: &str| otlp_span_filter(log_level).to_string(); + let directives = + |log_level: &str| openshell_otel::supervisor_span_filter(log_level).to_string(); assert_eq!(directives("warn"), "openshell=info,info"); assert_eq!(directives("info"), "openshell=info,info"); assert_eq!(directives("debug"), "openshell=debug,info"); diff --git a/docs/observability/ocsf-json-export.mdx b/docs/observability/ocsf-json-export.mdx index 96e30c75c0..db0faf4e2f 100644 --- a/docs/observability/ocsf-json-export.mdx +++ b/docs/observability/ocsf-json-export.mdx @@ -272,6 +272,64 @@ And a denied connection, produced when `curl https://github.com` runs in a sandb The JSON examples above are formatted for readability. The actual JSONL file contains one JSON object per line with no whitespace formatting. +## Follow a Deny Trace + +Automatic correlation adds `trace.uid` to events emitted inside a valid, +sampled OpenTelemetry span and `trace` to `metadata.profiles`. The UID is a W3C trace ID +with 32 lowercase hexadecimal characters. Search for it in your configured +Jaeger or Tempo backend to inspect the supervisor's handling of the deny. +Without an active sampled context, automatic correlation adds neither the +trace object nor its profile, including when OTLP export is disabled. +A sampled ID does not guarantee +successful export or retention in the trace backend. + +Official OCSF 1.8.0 includes the Trace profile on +[HTTP Activity](https://github.com/ocsf/ocsf-schema/blob/v1.8.0/events/network/http_activity.json) +and [API Activity](https://github.com/ocsf/ocsf-schema/blob/v1.8.0/events/application/api_activity.json). +OpenShell extends its applicability to Network Activity, Detection Finding, +Authentication, Base Event, SSH Activity, Process Activity, Application Lifecycle, +and Device Config State Change. This optional profile on those eight classes is +an OpenShell extension beyond the official 1.8.0 class definitions. Consumers +that enforce official profile applicability may reject these correlated records +unless configured to accept the extension. + +Supervisor deny correlation requires a dedicated supervisor configured for OTLP +trace export. Shared supervisor hosts currently do not initialize OTLP exporters, +so their sandbox audit records have no automatic trace correlation. + +```json +{ + "metadata": { "profiles": ["security_control", "trace"] }, + "trace": { "uid": "0af7651916cd43dd8448eb211c80319c" } +} +``` + +With OTLP export enabled, a network or HTTP deny creates an INFO +`supervisor.egress.deny` span around the audit records for that decision. +A domain event and its companion findings share one trace ID and one deny span. At the default supervisor +log level, the DEBUG connect span is disabled, so the trace contains only the +root deny span. It records the policy, reason and destination when available. +At debug level, a deny can share the connect trace and its authorization, +resolve and dial spans if those steps ran and the connect span remains +available. Allowed connections stay at DEBUG. Workload `traceparent` headers +do not supply supervisor correlation. + +Gateway audit records can inherit the caller's incoming `traceparent`, including +records emitted before authentication succeeds. Their trace IDs are correlation +hints, not evidence of the caller's identity or trust. An unsampled incoming +context can omit correlation. + +A deny span represents an enforced supervisor refusal, including fail-closed +middleware capacity refusals. When response middleware stops delivery, or the +proxy refuses to deliver a malformed upstream response head, it creates a deny +span. Ordinary upstream resets, downstream write failures, and response-body +framing errors do not create deny spans. Any delivery-failure findings describe +transport failure rather than an enforced refusal. +Allowed fail-open and audit-only traffic do not +create deny spans. Each refused decision can create an exported span, so repeated +refusals can increase telemetry volume; export and retention remain bounded by +your telemetry pipeline. + ## OCSF Event Classes in JSON The `class_uid` field identifies the event type: @@ -332,10 +390,12 @@ The setting takes effect on the next poll cycle, by default every 10 seconds. No Supported target versions are `1.1` and `1.3`. When set, the JSONL layer applies the following transformations: -- Strips fields added after the target version: `ai_model`, `container`, `observation_point_id` -- Removes unknown profile names from `metadata.profiles`: `ai_operation`, `container` +- Strips fields added after the target version: `ai_model`, `container`, `observation_point_id`, `trace` +- Removes unknown profile names from `metadata.profiles`: `ai_operation`, `container`, `trace` - Rewrites `metadata.version` to match the target version +The Trace profile first appears in OCSF 1.4. Both supported downgrade targets omit the top-level `trace` object and its profile. + The shorthand log output is unaffected. The internal event model stays at v1.8.0; only the serialized JSONL is transformed. When unset or empty, no downgrade is applied and events are written at the current schema version.