From 5190ab423f07f83639354bc977de1cdf2ae9c345 Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 13:35:29 -0400 Subject: [PATCH 01/17] docs(otlp): design spec for OTLP HTTP/protobuf trace export Records the approved design: vendor OTLP trace + collector protos and generate prost types (zero new runtime deps), keep the hand-rolled serde JSON path, share one mapper with a serde->prost converter, and select the protocol via builder + C FFI. Includes the dd-trace-py companion wiring and the layered E2E plan (local receiver, system-tests, sdk-backend-verify). Co-Authored-By: Claude Opus 4.8 --- ...-06-12-otlp-http-protobuf-export-design.md | 251 ++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 docs/superpowers/specs/2026-06-12-otlp-http-protobuf-export-design.md diff --git a/docs/superpowers/specs/2026-06-12-otlp-http-protobuf-export-design.md b/docs/superpowers/specs/2026-06-12-otlp-http-protobuf-export-design.md new file mode 100644 index 0000000000..8eb9c922a2 --- /dev/null +++ b/docs/superpowers/specs/2026-06-12-otlp-http-protobuf-export-design.md @@ -0,0 +1,251 @@ +# OTLP HTTP/protobuf trace export + +- **Date:** 2026-06-12 +- **Status:** Approved design, pending implementation plan +- **Repos:** `libdatadog` (feature), `dd-trace-py` (SDK wiring + E2E) +- **Branch (libdatadog):** `brian.marks/otlp-http-protobuf-export` + +## Background + +libdatadog can export traces over OTLP, but only as **HTTP/JSON**. The trace exporter +decodes incoming (msgpack) DD spans, maps them to an OTLP `ExportTraceServiceRequest`, serializes +that to JSON, and POSTs it with `Content-Type: application/json`. + +The groundwork for more encodings already exists: + +- `OtlpProtocol::{HttpJson, HttpProtobuf, Grpc}` is stubbed in `libdd-data-pipeline/src/otlp/config.rs` + (`HttpProtobuf` and `Grpc` carry `#[allow(dead_code)]` and "not supported yet"). +- The transport (`send_otlp_traces_http`) is format-agnostic: it POSTs a `Vec` body with a + content-type header and retries. The sidecar already POSTs `application/x-protobuf` for FFE metrics. +- `libdd-common::header::APPLICATION_PROTOBUF` (`application/x-protobuf`) already exists. +- `libdd-trace-protobuf` already vendors the OTLP `common/v1` and `resource/v1` protos and generates + Rust from them via `prost-build` + `protoc-bin-vendored` behind its `generate-protobuf` feature. +- The hand-rolled serde JSON types (`libdd-trace-utils/src/otlp_encoder/json_types.rs`) deliberately + duplicate the OTLP schema; the file comment anticipates a separate protobuf path. + +dd-trace-py is already pre-wired: it reads `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` / +`OTEL_EXPORTER_OTLP_PROTOCOL` into a `TRACES_PROTOCOL` setting (validated to `http/json` / +`http/protobuf`), exposes `TraceExporterBuilder` to Python via PyO3 with `set_otlp_endpoint` / +`set_otlp_headers`, and has a comment noting `TRACES_PROTOCOL` is "collected for telemetry but not +yet used to switch transport" because libdatadog only supports JSON. + +## Goal & scope + +Add OTLP **HTTP/protobuf** as a second trace-export encoding alongside HTTP/JSON, selectable per the +OTel-standard protocol values, and wire it through dd-trace-py so it is reachable from the SDK. + +**In scope** + +- Traces only. +- Encodings: `http/json` (existing) and `http/protobuf` (new). +- Protocol selection via the Rust builder, the C FFI, and dd-trace-py's Python builder + writer. +- Validation: Rust unit/integration tests, a dd-trace-py local E2E with a protobuf-decoding receiver, + system-tests against locally-built artifacts, and sdk-backend-verify against the Datadog backend. + +**Out of scope (non-goals)** — see "Non-goals / future" for where the design leaves room: + +- gRPC transport. +- gzip / `Content-Encoding`. +- OTLP `partial_success` response parsing. +- logs / metrics signals. + +## Decisions + +1. **Type source: vendor `.proto` + generate prost types** (not the `opentelemetry-proto` crate). + Rationale: `opentelemetry-proto 0.31` aligns with the workspace's prost 0.14, but its manifest makes + `opentelemetry` and `opentelemetry_sdk` non-optional and requires `tonic` + `tonic-prost` for the + message types — it drags the OTel Rust SDK and tonic into the widely-used `libdd-trace-utils`. For a + footprint-sensitive FFI library, vendoring the protos and generating prost types via the existing + `libdd-trace-protobuf` pipeline adds **zero new runtime dependencies** and follows an established + in-repo pattern. + +2. **Keep the hand-rolled serde JSON path; do not unify onto shared types.** + Rationale: OTLP/JSON deviates from canonical protobuf-JSON (trace/span IDs are hex, not base64; + int64 is a string). The hand-rolled serde types already implement this correctly and are tested. + Generating JSON from prost types (e.g. `pbjson`) would emit base64 IDs — wrong per the OTLP/JSON + spec. So the JSON path stays exactly as-is. + +3. **Share the mapping logic via one mapper + a mechanical converter.** + Rationale: the semantic DD-span→OTLP mapping (128-bit trace-id reconstruction, span-kind inference, + attribute limits, status, flags) runs once in `map_traces_to_otlp` and produces the serde types. The + protobuf path adds only a dumb, fully-tested structural converter from the serde types to the + generated prost types. No mapping logic is duplicated. + +## Architecture & data flow + +``` +DD spans (msgpack-decoded) + │ + ▼ +map_traces_to_otlp(...) ──► ExportTraceServiceRequest (hand-rolled serde types — UNCHANGED) + │ + ├─ HttpJson ─► serde_json::to_vec(&req) ─► Content-Type: application/json + └─ HttpProtobuf ─► (&req).into() : proto::Export…Request ─► prost encode_to_vec ─► application/x-protobuf + (mechanical serde→prost converter; no mapping logic duplicated) +``` + +The endpoint path (`/v1/traces`), retry strategy, sampling enforcement (unsampled chunks dropped +before export), and resource attributes are unchanged. + +## Component changes — libdatadog + +### A. `libdd-trace-protobuf` — vendor + generate the prost types + +- Add vendored protos under `src/pb/opentelemetry/proto/`: + - `trace/v1/trace.proto` + - `collector/trace/v1/trace_service.proto` (defines `ExportTraceServiceRequest`) +- Add both to the `compile_protos([...])` list in `build.rs` (alongside the existing common/resource + entries). +- Regenerate under `--features generate-protobuf` and commit the new `opentelemetry.proto.trace.v1.rs` + and `opentelemetry.proto.collector.trace.v1.rs` (matching the checked-in-generated convention). +- Net new external runtime deps: **zero** (`prost`, `prost-build`, `protoc-bin-vendored` already present). + +### B. `libdd-trace-utils::otlp_encoder` — converter + two encoders, feature-gated + +- `json_types.rs` and `mapper.rs`: **unchanged.** +- New `proto_convert.rs`: `impl From<&ExportTraceServiceRequest> for proto::ExportTraceServiceRequest`, + converting hex-string→16/8-byte IDs, int-string→i64, base64-string→bytes, the `AnyValue` enum→prost + `any_value::Value`, dropped counts, flags, status, links, events. Behind a new `otlp-protobuf` cargo + feature that pulls the generated types from `libdd-trace-protobuf`. +- `mod.rs` exposes: + - `encode_otlp_json(&req) -> serde_json::Result>` (always available), + - `encode_otlp_protobuf(&req) -> Vec` (feature-gated). +- The feature gate keeps non-OTLP and JSON-only consumers of `libdd-trace-utils` from paying for the + protobuf types. + +### C. `libdd-data-pipeline` — protocol dispatch + config plumbing + +- `otlp/config.rs`: make `OtlpProtocol` `pub`; add `impl FromStr` (`"http/json"→HttpJson`, + `"http/protobuf"→HttpProtobuf`, `"grpc"→Grpc`); drop `#[allow(dead_code)]` on `HttpProtobuf`. +- `otlp/exporter.rs` (`send_otlp_traces_http`): set content-type from `config.protocol` + (`APPLICATION_JSON` vs `APPLICATION_PROTOBUF`) instead of hardcoding JSON; rename `json_body`→`body`. +- `trace_exporter/mod.rs` (`send_otlp_traces_inner`): replace the hardcoded `serde_json::to_vec` with a + `match config.protocol` selecting `encode_otlp_json` / `encode_otlp_protobuf`. `Grpc` returns a clear + "not yet supported" `TraceExporterError`. +- `trace_exporter/builder.rs`: add `set_otlp_protocol(OtlpProtocol)`; use it where `OtlpProtocol::HttpJson` + is currently hardcoded. Enable the `otlp-protobuf` feature on the `libdd-trace-utils` dep. + +### D. `libdd-data-pipeline-ffi` — protocol setter + +- Add `otlp_protocol` to `TraceExporterConfig` and + `ddog_trace_exporter_config_set_otlp_protocol(config, CharSlice)` that parses the OTel string via + `FromStr`, rejecting `"grpc"` with `InvalidArgument` + a clear message. +- Apply it in the create fn next to `set_otlp_endpoint`. Regenerate the C header. + +## Protocol config surface + +Mirror the OTel SDK / dd-trace-java naming: callers pass `http/json` or `http/protobuf` (the values they +read from `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` / `OTEL_EXPORTER_OTLP_PROTOCOL`). libdatadog does not read +env vars itself — the host tracer resolves the value and calls the setter, consistent with +`set_otlp_endpoint`. + +**Default = `HttpJson`**, to preserve current behavior for existing integrations. (The OTel SDK and +dd-trace-java default to `http/protobuf`; keeping JSON the default here avoids changing behavior for +callers who don't set the protocol. Easy to flip later.) + +## Component changes — dd-trace-py (companion PR) + +1. **PyO3 binding** — `src/native/data_pipeline/mod.rs`: add `set_otlp_protocol(&str)` forwarding to the + new builder method. +2. **Writer wiring** — `ddtrace/internal/writer/writer.py` `_create_exporter()`: call + `builder.set_otlp_protocol(otel_config.exporter.TRACES_PROTOCOL)` when OTLP is enabled. +3. **Un-stub the comments** — drop the "not yet used to switch transport" note at + `ddtrace/internal/settings/_opentelemetry.py` and the "libdatadog currently only supports http/json" + default note. +4. **Cargo dependency** — once libdatadog ships a release containing this feature, bump the + `rev = "v35.0.0"` git pins in `src/native/Cargo.toml`. Until then, the local cargo patch (below) is + used for E2E. + +The dd-trace-py PR is only mergeable after a libdatadog release contains the feature; it is sequenced +after the libdatadog PR. + +## Testing strategy — libdatadog + +- Existing JSON snapshot test (`otlp_export_sends_correct_payload`) and all `mapper.rs` unit tests stay + green, unchanged (JSON path untouched). +- New `proto_convert` unit tests: serde→prost equivalence (trace/span/parent IDs as bytes, kind, status, + all `AnyValue` variants incl. bytes/array, dropped counts, flags, links, events). +- New protobuf export integration test (mirrors the JSON one): mock server asserts + `Content-Type: application/x-protobuf` + path `/v1/traces`, then prost-decodes the body and asserts + `resource_spans` / `service.name` / span names. +- New parity test: `map → encode_json` vs `map → encode_protobuf → prost-decode` carry identical data — + guards the two encoders against drift. +- `FromStr` + FFI-setter tests (including `grpc` rejection). +- `cargo ffi-test` (C/C++ examples) since FFI signatures change. + +## E2E validation + +Layered, from fastest/most-deterministic to fullest-chain. + +### Tier 1 — dd-trace-py local receiver (deterministic, repeatable) + +- Point dd-trace-py at the local libdatadog build via a git-keyed cargo patch in `src/native/` + (the deps are git deps, so this is **not** `[patch.crates-io]`): + + ```toml + [patch."https://github.com/DataDog/libdatadog"] + libdd-data-pipeline = { path = "/path/to/local/libdatadog/libdd-data-pipeline" } + libdd-trace-utils = { path = "/path/to/local/libdatadog/libdd-trace-utils" } + libdd-trace-protobuf = { path = "/path/to/local/libdatadog/libdd-trace-protobuf" } + # + any other crate in the modified set + ``` + + dd-trace-py builds use libdatadog's committed generated prost code, so no `protoc` is needed there. + +- Build dd-trace-py in a fresh venv (`pip install -e .`). +- Run a small local OTLP/HTTP receiver on `:4318` handling `POST /v1/traces`: assert + `Content-Type: application/x-protobuf`, `ExportTraceServiceRequest().ParseFromString(body)` with the + `opentelemetry-proto` Python package, and assert resource `service.name`, span names, and the + 32-hex-char `trace_id` survive the round trip. +- Run a tiny instrumented app twice — `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf` and `http/json` + — confirming the new path works and the existing JSON path is unaffected. Ensure + `DD_TRACE_AGENT_PROTOCOL_VERSION` is unset (it disables OTLP). + +### Tier 2 — system-tests against local builds (via `apm-ecosystems:system-tests-local`) + +- Build dd-trace-py against the local libdatadog (Tier 1 patch), then run the relevant system-tests + OTLP scenario(s) with the locally-built tracer. The exact scenario / parametric test name is to be + identified during planning. Goal: exercise the protobuf path through the supported system-tests + harness rather than only a bespoke receiver. + +### Tier 3 — sdk-backend-verify (full chain to the Datadog backend, via `apm-ecosystems:sdk-backend-verify`) + +- Run the instrumented app with `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf` against an OTLP + receiver that forwards to the Datadog backend (DD Agent OTLP intake on `:4318`, or the OTel Collector + with a Datadog exporter), then verify the spans land in the backend with correct service/resource/ + trace-id via the backend APIs. Confirms the protobuf bytes are accepted end-to-end and ingested. + +## Validation gauntlet (per AGENTS.md) + +For each touched crate: `cargo check -p ` → +`cargo +nightly-2026-02-08 fmt --all -- --check` → +`cargo +stable clippy --workspace --all-targets --all-features -- -D warnings` → +`cargo nextest run` (workspace + all-features) → `cargo test --doc` → `cargo ffi-test`. +If `Cargo.lock` changes: `./scripts/update_license_3rdparty.sh` + `cargo deny check`. +Apache headers on new files via `./scripts/reformat_copyright.sh`. + +## Risks & mitigations + +- **Footprint spike (Phase 0 gate):** before real work, add the vendored protos, regenerate, and confirm + `cargo tree -p libdd-trace-utils --features otlp-protobuf` shows no new heavy crates. This is the whole + premise of decision 1 — go/no-go. +- **Converter correctness** (hex/base64/int-string round-trips): covered by the parity and converter + unit tests. +- **proto3 field presence:** prost uses `0`/empty for absent scalars; the converter must map + `Option`/empty consistently. Covered by unit tests; semantically harmless for OTLP receivers. +- **Cross-repo sequencing:** the dd-trace-py PR depends on a libdatadog release. E2E uses the local + cargo patch until then; the PR documents the required version bump. + +## Non-goals / future hooks + +- **gRPC:** `OtlpProtocol::Grpc` stays; rejected at the setter/exporter. A future addition is isolated to + the exporter plus a transport that doesn't fit today's HTTP/1 client. +- **gzip:** add later as a `Content-Encoding` on the existing body (`flate2` is already available). +- **`partial_success`:** neither dd-trace-go nor dd-trace-java parse it; keep status-only handling. + +## Sequencing / PR plan + +1. **libdatadog PR** (this branch): feature + unit/integration tests + regenerated protos + C header. +2. **Local E2E** (Tier 1) against the libdatadog branch via cargo patch in a dd-trace-py worktree. +3. **dd-trace-py PR**: PyO3 binding + writer wiring + comment cleanup; depends on a libdatadog release + bump. Validated with system-tests (Tier 2) and sdk-backend-verify (Tier 3) against local builds. From 946f116d58ef4771108da041f6249e3f962a0cbf Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 13:40:58 -0400 Subject: [PATCH 02/17] docs(otlp): implementation plan for OTLP HTTP/protobuf trace export Bite-sized, TDD-structured plan across 9 phases: vendor+generate prost types, serde->prost converter, encoder dispatch, protocol config through builder + C FFI, full validation gauntlet + libdatadog PR, then dd-trace-py PyO3/writer wiring and three E2E tiers (local receiver, system-tests, sdk-backend-verify) + dd-trace-py PR. Co-Authored-By: Claude Opus 4.8 --- .../2026-06-12-otlp-http-protobuf-export.md | 1199 +++++++++++++++++ 1 file changed, 1199 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-12-otlp-http-protobuf-export.md diff --git a/docs/superpowers/plans/2026-06-12-otlp-http-protobuf-export.md b/docs/superpowers/plans/2026-06-12-otlp-http-protobuf-export.md new file mode 100644 index 0000000000..90e865bb27 --- /dev/null +++ b/docs/superpowers/plans/2026-06-12-otlp-http-protobuf-export.md @@ -0,0 +1,1199 @@ +# OTLP HTTP/protobuf trace export — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add OTLP HTTP/protobuf as a second trace-export encoding alongside HTTP/JSON in libdatadog, selectable via the OTel-standard protocol values, and wire it through dd-trace-py with end-to-end validation. + +**Architecture:** Vendor the OTLP `trace` + `collector/trace` protos into `libdd-trace-protobuf` and generate prost types (zero new runtime deps). Keep the existing hand-rolled serde JSON path untouched. The semantic DD-span→OTLP mapping runs once and produces the serde types; a mechanical `From<&serde_types>` converter produces the prost types for protobuf. The exporter selects encoder + content-type from `OtlpProtocol`. dd-trace-py gains a `set_otlp_protocol` binding and passes its already-parsed `TRACES_PROTOCOL` through. + +**Tech Stack:** Rust (prost 0.14, prost-build, protoc-bin-vendored, serde_json, httpmock), C FFI (cbindgen), Python/PyO3 (setuptools-rust), system-tests, sdk-backend-verify. + +**Spec:** `docs/superpowers/specs/2026-06-12-otlp-http-protobuf-export-design.md` + +**Refinement vs spec:** The spec proposed gating the protobuf encoder behind an `otlp-protobuf` cargo feature. During planning we confirmed the generated OTLP types live in `libdd-trace-protobuf` (already a non-optional dep of `libdd-trace-utils`) and, matching the existing OTLP common/resource pattern, are compiled unconditionally. A feature gate would only guard the small converter module for negligible benefit, so this plan drops the gate (YAGNI). No new runtime dependency is introduced either way. + +**Worktrees:** +- libdatadog: `/Users/brian.marks/go/src/github.com/DataDog/libdatadog-otlp-http-protobuf-export` (branch `brian.marks/otlp-http-protobuf-export`) — already created. +- dd-trace-py: create at execution time (Phase 6). + +--- + +## Phase 0 — Footprint spike (go/no-go gate) + +### Task 0: Confirm vendored prost types add no heavy dependencies + +**Files:** none (investigation). + +- [ ] **Step 1: Record the OTel proto version already vendored** + +Run: +```bash +cd /Users/brian.marks/go/src/github.com/DataDog/libdatadog-otlp-http-protobuf-export +head -20 libdd-trace-protobuf/src/pb/opentelemetry/proto/common/v1/common.proto +``` +Expected: a header/comment indicating the upstream opentelemetry-proto version (e.g. a release tag or proto package version). Note this version — Phase 1 vendors `trace.proto` + `trace_service.proto` from the **same** release for import compatibility. + +- [ ] **Step 2: Confirm prost is already the protobuf toolchain (no new runtime crate needed)** + +Run: +```bash +grep -n 'prost' libdd-trace-protobuf/Cargo.toml libdd-trace-utils/Cargo.toml +``` +Expected: `prost = "0.14.x"` present in both; `prost-build` + `protoc-bin-vendored` present in `libdd-trace-protobuf` under `[build-dependencies]` behind `generate-protobuf`. Conclusion: vendoring adds only generated structs, no new external runtime crate. + +- [ ] **Step 3: Gate decision** + +If Steps 1–2 hold (they should, per the spec's prior investigation), proceed. If a new heavy crate would be required, STOP and revisit the spec's decision 1. + +--- + +## Phase 1 — Generate OTLP trace + collector prost types (`libdd-trace-protobuf`) + +### Task 1: Vendor the OTLP trace + collector protos and generate prost types + +**Files:** +- Create: `libdd-trace-protobuf/src/pb/opentelemetry/proto/trace/v1/trace.proto` +- Create: `libdd-trace-protobuf/src/pb/opentelemetry/proto/collector/trace/v1/trace_service.proto` +- Modify: `libdd-trace-protobuf/build.rs` (compile list + license prepend) +- Create (generated, committed): `libdd-trace-protobuf/src/opentelemetry.proto.trace.v1.rs`, `libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs` + +- [ ] **Step 1: Vendor the two proto files from the matching opentelemetry-proto release** + +Use the same release tag noted in Task 0. From the opentelemetry-proto repo, copy verbatim: +- `opentelemetry/proto/trace/v1/trace.proto` → `libdd-trace-protobuf/src/pb/opentelemetry/proto/trace/v1/trace.proto` +- `opentelemetry/proto/collector/trace/v1/trace_service.proto` → `libdd-trace-protobuf/src/pb/opentelemetry/proto/collector/trace/v1/trace_service.proto` + +```bash +mkdir -p libdd-trace-protobuf/src/pb/opentelemetry/proto/trace/v1 +mkdir -p libdd-trace-protobuf/src/pb/opentelemetry/proto/collector/trace/v1 +TAG= # e.g. v1.5.0 +BASE="https://raw.githubusercontent.com/open-telemetry/opentelemetry-proto/$TAG/opentelemetry/proto" +curl -fsSL "$BASE/trace/v1/trace.proto" -o libdd-trace-protobuf/src/pb/opentelemetry/proto/trace/v1/trace.proto +curl -fsSL "$BASE/collector/trace/v1/trace_service.proto" -o libdd-trace-protobuf/src/pb/opentelemetry/proto/collector/trace/v1/trace_service.proto +``` +Expected: both files saved. `trace.proto` imports `opentelemetry/proto/common/v1/common.proto` and `.../resource/v1/resource.proto` (already vendored). `trace_service.proto` imports `.../trace/v1/trace.proto` and defines `ExportTraceServiceRequest`/`ExportTraceServiceResponse`. + +- [ ] **Step 2: Add both protos to the compile list in `build.rs`** + +In `libdd-trace-protobuf/build.rs`, extend the `compile_protos(&[ ... ], &["src/pb/"])` array (currently ending at `"src/pb/idx/span.proto"`): + +```rust + &[ + "src/pb/agent_payload.proto", + "src/pb/tracer_payload.proto", + "src/pb/span.proto", + "src/pb/stats.proto", + "src/pb/remoteconfig.proto", + "src/pb/opentelemetry/proto/common/v1/process_context.proto", + "src/pb/opentelemetry/proto/trace/v1/trace.proto", + "src/pb/opentelemetry/proto/collector/trace/v1/trace_service.proto", + "src/pb/idx/tracer_payload.proto", + "src/pb/idx/span.proto", + ], +``` + +- [ ] **Step 3: Prepend the OTel license header to the new generated files** + +In `build.rs`, next to the existing `prepend_to_file(otel_license, ...resource.v1.rs)` / `...common.v1.rs` calls, add: + +```rust + prepend_to_file( + otel_license, + &output_path.join("opentelemetry.proto.trace.v1.rs"), + ); + prepend_to_file( + otel_license, + &output_path.join("opentelemetry.proto.collector.trace.v1.rs"), + ); +``` + +- [ ] **Step 4: Regenerate the committed Rust types** + +Run: +```bash +cargo build -p libdd-trace-protobuf --features generate-protobuf +``` +Expected: build succeeds; new files `libdd-trace-protobuf/src/opentelemetry.proto.trace.v1.rs` and `libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs` appear, and `libdd-trace-protobuf/src/_includes.rs` now references the `opentelemetry::proto::trace::v1` and `opentelemetry::proto::collector::trace::v1` modules. + +- [ ] **Step 5: Verify the generated type path compiles and is reachable** + +Run: +```bash +cargo build -p libdd-trace-protobuf +``` +Then confirm the symbol path with a throwaway check: +```bash +grep -rn "ExportTraceServiceRequest" libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs | head +``` +Expected: `pub struct ExportTraceServiceRequest` present. Its module path is `libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest`, with span/resource types under `opentelemetry::proto::trace::v1` and `...::common::v1` / `...::resource::v1`. + +- [ ] **Step 6: Commit** + +```bash +git add libdd-trace-protobuf/src/pb/opentelemetry libdd-trace-protobuf/build.rs \ + libdd-trace-protobuf/src/opentelemetry.proto.trace.v1.rs \ + libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs \ + libdd-trace-protobuf/src/_includes.rs +git commit -m "feat(trace-protobuf): vendor + generate OTLP trace/collector prost types" +``` + +--- + +## Phase 2 — Converter + protobuf encoder (`libdd-trace-utils::otlp_encoder`) + +Module paths below assume the generated types are re-exported as +`libdd_trace_protobuf::opentelemetry::proto::{trace::v1 as otlp_trace, common::v1 as otlp_common, resource::v1 as otlp_resource, collector::trace::v1 as otlp_collector}`. Confirm exact paths from Task 1 Step 5 and adjust the `use` lines if the generated module nesting differs. + +### Task 2: serde→prost converter for the OTLP request + +**Files:** +- Create: `libdd-trace-utils/src/otlp_encoder/proto_convert.rs` +- Modify: `libdd-trace-utils/src/otlp_encoder/mod.rs` (declare module + re-export encoders) +- Test: inline `#[cfg(test)]` in `proto_convert.rs` + +- [ ] **Step 1: Write the failing test for the converter** + +Create `libdd-trace-utils/src/otlp_encoder/proto_convert.rs` with only the test module first: + +```rust +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(test)] +mod tests { + use crate::otlp_encoder::{map_traces_to_otlp, OtlpResourceInfo}; + use crate::span::BytesData; + use crate::span::v04::Span; + use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest as ProtoReq; + + #[test] + fn converts_ids_and_attributes_to_proto() { + let resource_info = OtlpResourceInfo { + service: "svc".to_string(), + ..Default::default() + }; + let mut span: Span = Span { + trace_id: 0xD269B633813FC60C_u128, + span_id: 0xEEE19B7EC3C1B174, + parent_id: 0xEEE19B7EC3C1B173, + name: libdd_tinybytes::BytesString::from_static("op"), + resource: libdd_tinybytes::BytesString::from_static("res"), + r#type: libdd_tinybytes::BytesString::from_static("web"), + start: 1544712660000000000, + duration: 1000000000, + error: 0, + ..Default::default() + }; + span.metrics + .insert(libdd_tinybytes::BytesString::from_static("count"), 42.0); + + let serde_req = map_traces_to_otlp(vec![vec![span]], &resource_info); + let proto: ProtoReq = (&serde_req).into(); + + let rs = &proto.resource_spans[0]; + let sp = &rs.scope_spans[0].spans[0]; + // trace_id: 16 bytes, big-endian, high 64 bits zero (no _dd.p.tid) + assert_eq!( + sp.trace_id, + vec![0, 0, 0, 0, 0, 0, 0, 0, 0xD2, 0x69, 0xB6, 0x33, 0x81, 0x3F, 0xC6, 0x0C] + ); + assert_eq!(sp.span_id, vec![0xEE, 0xE1, 0x9B, 0x7E, 0xC3, 0xC1, 0xB1, 0x74]); + assert_eq!(sp.parent_span_id, vec![0xEE, 0xE1, 0x9B, 0x7E, 0xC3, 0xC1, 0xB1, 0x73]); + assert_eq!(sp.name, "res"); + assert_eq!(sp.start_time_unix_nano, 1544712660000000000); + assert_eq!(sp.end_time_unix_nano, 1544712661000000000); + // count metric -> int attribute + let count = sp + .attributes + .iter() + .find(|kv| kv.key == "count") + .expect("count attr"); + use libdd_trace_protobuf::opentelemetry::proto::common::v1::any_value::Value; + assert!(matches!(count.value.as_ref().unwrap().value, Some(Value::IntValue(42)))); + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails to compile (no `From` impl yet)** + +Run: +```bash +cargo test -p libdd-trace-utils otlp_encoder::proto_convert -- --nocapture +``` +Expected: compile error — `From<&ExportTraceServiceRequest>` not implemented / trait bound not satisfied. + +- [ ] **Step 3: Implement the converter** + +Prepend the implementation above the test module in `proto_convert.rs`. Use the generated module paths confirmed in Task 1 Step 5: + +```rust +//! Converts the hand-rolled serde OTLP request (the JSON wire model) into the generated +//! prost types for binary (HTTP/protobuf) export. The semantic DD-span -> OTLP mapping already +//! happened in `mapper.rs`; this is a purely structural translation. + +use crate::otlp_encoder::json_types as j; +use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest as ProtoReq; +use libdd_trace_protobuf::opentelemetry::proto::common::v1::{ + any_value::Value as ProtoValue, AnyValue as ProtoAnyValue, ArrayValue as ProtoArrayValue, + InstrumentationScope as ProtoScope, KeyValue as ProtoKeyValue, +}; +use libdd_trace_protobuf::opentelemetry::proto::resource::v1::Resource as ProtoResource; +use libdd_trace_protobuf::opentelemetry::proto::trace::v1::{ + span::{Event as ProtoEvent, Link as ProtoLink}, + status::StatusCode as ProtoStatusCode, + ResourceSpans as ProtoResourceSpans, ScopeSpans as ProtoScopeSpans, Span as ProtoSpan, + Status as ProtoStatus, +}; + +/// Decode a fixed-width lowercase hex string into a byte vector. The mapper always produces +/// well-formed hex of the expected width; on the unexpected event of a malformed value we fall +/// back to an all-zero buffer of `len` bytes rather than panicking (FFI reliability). +fn hex_to_bytes(s: &str, len: usize) -> Vec { + let mut out = Vec::with_capacity(len); + let bytes = s.as_bytes(); + if bytes.len() == len * 2 { + let mut i = 0; + while i < bytes.len() { + match (hex_nibble(bytes[i]), hex_nibble(bytes[i + 1])) { + (Some(hi), Some(lo)) => out.push((hi << 4) | lo), + _ => return vec![0u8; len], + } + i += 2; + } + out + } else { + vec![0u8; len] + } +} + +fn hex_nibble(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +fn parse_u64(s: &str) -> u64 { + s.parse().unwrap_or(0) +} + +impl From<&j::AnyValue> for ProtoAnyValue { + fn from(v: &j::AnyValue) -> Self { + let value = match v { + j::AnyValue::StringValue(s) => ProtoValue::StringValue(s.clone()), + j::AnyValue::BoolValue(b) => ProtoValue::BoolValue(*b), + j::AnyValue::IntValue(i) => ProtoValue::IntValue(*i), + j::AnyValue::DoubleValue(d) => ProtoValue::DoubleValue(*d), + j::AnyValue::BytesValue(b) => ProtoValue::BytesValue(b.clone()), + j::AnyValue::ArrayValue(a) => ProtoValue::ArrayValue(ProtoArrayValue { + values: a.values.iter().map(ProtoAnyValue::from).collect(), + }), + }; + ProtoAnyValue { value: Some(value) } + } +} + +fn kv(k: &j::KeyValue) -> ProtoKeyValue { + ProtoKeyValue { + key: k.key.clone(), + value: Some(ProtoAnyValue::from(&k.value)), + } +} + +impl From<&j::ExportTraceServiceRequest> for ProtoReq { + fn from(req: &j::ExportTraceServiceRequest) -> Self { + ProtoReq { + resource_spans: req.resource_spans.iter().map(resource_spans).collect(), + } + } +} + +fn resource_spans(rs: &j::ResourceSpans) -> ProtoResourceSpans { + ProtoResourceSpans { + resource: rs.resource.as_ref().map(|r| ProtoResource { + attributes: r.attributes.iter().map(kv).collect(), + dropped_attributes_count: 0, + }), + scope_spans: rs.scope_spans.iter().map(scope_spans).collect(), + schema_url: String::new(), + } +} + +fn scope_spans(ss: &j::ScopeSpans) -> ProtoScopeSpans { + ProtoScopeSpans { + scope: ss.scope.as_ref().map(|s| ProtoScope { + name: s.name.clone().unwrap_or_default(), + version: s.version.clone().unwrap_or_default(), + attributes: Vec::new(), + dropped_attributes_count: 0, + }), + spans: ss.spans.iter().map(span).collect(), + schema_url: ss.schema_url.clone().unwrap_or_default(), + } +} + +fn span(s: &j::OtlpSpan) -> ProtoSpan { + ProtoSpan { + trace_id: hex_to_bytes(&s.trace_id, 16), + span_id: hex_to_bytes(&s.span_id, 8), + trace_state: s.trace_state.clone().unwrap_or_default(), + parent_span_id: s + .parent_span_id + .as_ref() + .map(|p| hex_to_bytes(p, 8)) + .unwrap_or_default(), + flags: s.flags.unwrap_or(0), + name: s.name.clone(), + kind: s.kind, + start_time_unix_nano: parse_u64(&s.start_time_unix_nano), + end_time_unix_nano: parse_u64(&s.end_time_unix_nano), + attributes: s.attributes.iter().map(kv).collect(), + dropped_attributes_count: s.dropped_attributes_count.unwrap_or(0), + events: s.events.iter().map(event).collect(), + dropped_events_count: s.dropped_events_count.unwrap_or(0), + links: s.links.iter().map(link).collect(), + dropped_links_count: 0, + status: Some(ProtoStatus { + message: s.status.message.clone().unwrap_or_default(), + code: status_code(s.status.code), + }), + } +} + +fn status_code(code: i32) -> i32 { + // Mirror j::status_code constants onto the generated enum's i32 values. + match code { + c if c == j::status_code::OK => ProtoStatusCode::Ok as i32, + c if c == j::status_code::ERROR => ProtoStatusCode::Error as i32, + _ => ProtoStatusCode::Unset as i32, + } +} + +fn link(l: &j::OtlpSpanLink) -> ProtoLink { + ProtoLink { + trace_id: hex_to_bytes(&l.trace_id, 16), + span_id: hex_to_bytes(&l.span_id, 8), + trace_state: l.trace_state.clone().unwrap_or_default(), + attributes: l.attributes.iter().map(kv).collect(), + dropped_attributes_count: l.dropped_attributes_count.unwrap_or(0), + flags: 0, + } +} + +fn event(e: &j::OtlpSpanEvent) -> ProtoEvent { + ProtoEvent { + time_unix_nano: parse_u64(&e.time_unix_nano), + name: e.name.clone(), + attributes: e.attributes.iter().map(kv).collect(), + dropped_attributes_count: e.dropped_attributes_count.unwrap_or(0), + } +} +``` + +> Note: field names/struct shapes above are the standard prost OTLP output, but prost can name nested types differently across versions. After generation (Task 1), open `opentelemetry.proto.trace.v1.rs` and reconcile any field names (`dropped_links_count`, `flags`, the `span::{Event, Link}` / `status::StatusCode` nesting) with the generated source before finishing this task. + +- [ ] **Step 4: Declare the module in `mod.rs`** + +In `libdd-trace-utils/src/otlp_encoder/mod.rs`, add under the existing `pub mod mapper;`: +```rust +pub mod proto_convert; +``` + +- [ ] **Step 5: Run the converter test to verify it passes** + +Run: +```bash +cargo test -p libdd-trace-utils otlp_encoder::proto_convert -- --nocapture +``` +Expected: PASS. If field-name mismatches appear, fix per the note in Step 3, then re-run. + +- [ ] **Step 6: Commit** + +```bash +git add libdd-trace-utils/src/otlp_encoder/proto_convert.rs libdd-trace-utils/src/otlp_encoder/mod.rs +git commit -m "feat(trace-utils): add serde->prost OTLP converter" +``` + +### Task 3: Public encoders (`encode_otlp_json`, `encode_otlp_protobuf`) + parity test + +**Files:** +- Modify: `libdd-trace-utils/src/otlp_encoder/mod.rs` +- Test: inline `#[cfg(test)]` in `mod.rs` + +- [ ] **Step 1: Write the failing parity test** + +Add to `libdd-trace-utils/src/otlp_encoder/mod.rs` a test module: + +```rust +#[cfg(test)] +mod encode_tests { + use super::*; + use crate::span::BytesData; + use crate::span::v04::Span; + use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest as ProtoReq; + use prost::Message; + + fn sample() -> ExportTraceServiceRequest { + let resource_info = OtlpResourceInfo { service: "svc".to_string(), ..Default::default() }; + let span: Span = Span { + trace_id: 0xD269B633813FC60C_u128, + span_id: 0xEEE19B7EC3C1B174, + name: libdd_tinybytes::BytesString::from_static("op"), + resource: libdd_tinybytes::BytesString::from_static("res"), + start: 1, duration: 2, ..Default::default() + }; + map_traces_to_otlp(vec![vec![span]], &resource_info) + } + + #[test] + fn json_and_protobuf_carry_same_span() { + let req = sample(); + let json = encode_otlp_json(&req).unwrap(); + let pb = encode_otlp_protobuf(&req); + + let json_v: serde_json::Value = serde_json::from_slice(&json).unwrap(); + let json_name = json_v["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["name"] + .as_str().unwrap().to_string(); + + let proto = ProtoReq::decode(pb.as_slice()).unwrap(); + let proto_name = proto.resource_spans[0].scope_spans[0].spans[0].name.clone(); + + assert_eq!(json_name, "res"); + assert_eq!(proto_name, "res"); + // Span id round-trips identically: JSON hex vs proto bytes. + let json_sid = json_v["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["spanId"] + .as_str().unwrap().to_string(); + let proto_sid = &proto.resource_spans[0].scope_spans[0].spans[0].span_id; + assert_eq!(json_sid, hex::encode(proto_sid)); + } +} +``` + +Add `hex = "0.4"` to `libdd-trace-utils` `[dev-dependencies]` if not already present (used only in tests). + +- [ ] **Step 2: Run to verify it fails (encoders not defined)** + +Run: +```bash +cargo test -p libdd-trace-utils otlp_encoder::encode_tests -- --nocapture +``` +Expected: compile error — `encode_otlp_json` / `encode_otlp_protobuf` not found. + +- [ ] **Step 3: Implement the encoders** + +Add to `libdd-trace-utils/src/otlp_encoder/mod.rs` (after the `pub use mapper::map_traces_to_otlp;` line): + +```rust +use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest as ProtoExportTraceServiceRequest; +use prost::Message; + +pub use json_types::ExportTraceServiceRequest; + +/// Serialize an OTLP request to the HTTP/JSON wire format. +pub fn encode_otlp_json( + req: &ExportTraceServiceRequest, +) -> serde_json::Result> { + serde_json::to_vec(req) +} + +/// Serialize an OTLP request to the HTTP/protobuf wire format. +pub fn encode_otlp_protobuf(req: &ExportTraceServiceRequest) -> Vec { + let proto: ProtoExportTraceServiceRequest = req.into(); + proto.encode_to_vec() +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: +```bash +cargo test -p libdd-trace-utils otlp_encoder:: -- --nocapture +``` +Expected: PASS (parity test + converter test + existing mapper tests all green). + +- [ ] **Step 5: Commit** + +```bash +git add libdd-trace-utils/src/otlp_encoder/mod.rs libdd-trace-utils/Cargo.toml +git commit -m "feat(trace-utils): add encode_otlp_json/encode_otlp_protobuf" +``` + +--- + +## Phase 3 — Protocol selection + dispatch (`libdd-data-pipeline`) + +### Task 4: Make `OtlpProtocol` public + `FromStr` + +**Files:** +- Modify: `libdd-data-pipeline/src/otlp/config.rs` +- Test: inline `#[cfg(test)]` in `config.rs` + +- [ ] **Step 1: Write the failing test** + +Add to `libdd-data-pipeline/src/otlp/config.rs`: + +```rust +#[cfg(test)] +mod tests { + use super::*; + use std::str::FromStr; + + #[test] + fn protocol_from_str() { + assert_eq!(OtlpProtocol::from_str("http/json").unwrap(), OtlpProtocol::HttpJson); + assert_eq!(OtlpProtocol::from_str("http/protobuf").unwrap(), OtlpProtocol::HttpProtobuf); + assert_eq!(OtlpProtocol::from_str("grpc").unwrap(), OtlpProtocol::Grpc); + assert!(OtlpProtocol::from_str("nonsense").is_err()); + } +} +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cargo test -p libdd-data-pipeline otlp::config -- --nocapture` +Expected: compile error — `from_str` not implemented; `OtlpProtocol` not public. + +- [ ] **Step 3: Implement** + +In `libdd-data-pipeline/src/otlp/config.rs` change the enum visibility and remove the dead-code allow on `HttpProtobuf`: + +```rust +/// OTLP trace export protocol. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum OtlpProtocol { + /// HTTP with JSON body (Content-Type: application/json). Default for HTTP. + #[default] + HttpJson, + /// HTTP with protobuf body (Content-Type: application/x-protobuf). + HttpProtobuf, + /// gRPC. (Not supported yet) + #[allow(dead_code)] + Grpc, +} + +impl std::str::FromStr for OtlpProtocol { + type Err = String; + fn from_str(s: &str) -> Result { + match s { + "http/json" => Ok(OtlpProtocol::HttpJson), + "http/protobuf" => Ok(OtlpProtocol::HttpProtobuf), + "grpc" => Ok(OtlpProtocol::Grpc), + other => Err(format!("unknown OTLP protocol: {other}")), + } + } +} +``` +Also change `protocol: OtlpProtocol` field on `OtlpTraceConfig` from `pub(crate)` to `pub` and drop its `#[allow(dead_code)]`. + +- [ ] **Step 4: Run to verify it passes** + +Run: `cargo test -p libdd-data-pipeline otlp::config -- --nocapture` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add libdd-data-pipeline/src/otlp/config.rs +git commit -m "feat(data-pipeline): make OtlpProtocol public with FromStr" +``` + +### Task 5: Content-type by protocol in the transport + +**Files:** +- Modify: `libdd-data-pipeline/src/otlp/exporter.rs` + +- [ ] **Step 1: Update `send_otlp_traces_http` to choose content-type from protocol** + +In `libdd-data-pipeline/src/otlp/exporter.rs`, rename the `json_body: Vec` parameter to `body: Vec`, pass it to `send_with_retry` instead of `json_body`, and replace the hardcoded content-type insert: + +```rust + let content_type = match config.protocol { + crate::otlp::config::OtlpProtocol::HttpProtobuf => libdd_common::header::APPLICATION_PROTOBUF, + _ => libdd_common::header::APPLICATION_JSON, + }; + let mut headers = config.headers.clone(); + headers.insert(http::header::CONTENT_TYPE, content_type); +``` + +- [ ] **Step 2: Verify it compiles** + +Run: `cargo check -p libdd-data-pipeline` +Expected: success (the caller still passes JSON bytes; updated in Task 6). + +- [ ] **Step 3: Commit** + +```bash +git add libdd-data-pipeline/src/otlp/exporter.rs +git commit -m "feat(data-pipeline): set OTLP content-type from protocol" +``` + +### Task 6: Encoder dispatch in the send path + +**Files:** +- Modify: `libdd-data-pipeline/src/trace_exporter/mod.rs` (`send_otlp_traces_inner`, ~line 548) +- Modify: `libdd-data-pipeline/src/trace_exporter/mod.rs` imports (~line 18) + +- [ ] **Step 1: Replace the hardcoded JSON serialization with protocol dispatch** + +In `send_otlp_traces_inner`, replace the `serde_json::to_vec(&request)` block with: + +```rust + let request = map_traces_to_otlp(traces, &resource_info); + let body = match config.protocol { + OtlpProtocol::HttpJson => { + libdd_trace_utils::otlp_encoder::encode_otlp_json(&request).map_err(|e| { + error!("OTLP JSON serialization error: {e}"); + TraceExporterError::Internal(InternalErrorKind::InvalidWorkerState(e.to_string())) + })? + } + OtlpProtocol::HttpProtobuf => { + libdd_trace_utils::otlp_encoder::encode_otlp_protobuf(&request) + } + OtlpProtocol::Grpc => { + return Err(TraceExporterError::Internal(InternalErrorKind::InvalidWorkerState( + "OTLP gRPC export is not supported".to_string(), + ))); + } + }; + send_otlp_traces_http( + &self.capabilities, + config, + self.endpoint.test_token.as_deref(), + body, + ) + .await?; +``` + +Add `OtlpProtocol` to the `use crate::otlp::{...}` import line at the top of the file. + +- [ ] **Step 2: Verify the workspace builds** + +Run: `cargo check -p libdd-data-pipeline` +Expected: success. + +- [ ] **Step 3: Add a protobuf export integration test** + +Create `libdd-data-pipeline/tests/test_trace_exporter_otlp_protobuf_export.rs`: + +```rust +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 +#[cfg(test)] +mod otlp_protobuf_tests { + use libdd_capabilities_impl::NativeCapabilities; + use libdd_data_pipeline::trace_exporter::TraceExporterBuilder; + use libdd_trace_utils::test_utils::create_test_json_span; + use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest; + use prost::Message; + use serde_json::json; + use tokio::task; + + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn otlp_protobuf_export_sends_decodable_payload() { + use httpmock::MockServer; + let server = MockServer::start_async().await; + let mut mock = server + .mock_async(|when, then| { + when.method("POST") + .path("/v1/traces") + .header("content-type", "application/x-protobuf"); + then.status(200).body(""); + }) + .await; + + let endpoint = format!("http://localhost:{}/v1/traces", server.port()); + let task_result = task::spawn_blocking(move || { + let mut builder = TraceExporterBuilder::default(); + builder + .set_otlp_endpoint(&endpoint) + .set_otlp_protocol(libdd_data_pipeline::otlp::config::OtlpProtocol::HttpProtobuf) + .set_language("test-lang") + .set_tracer_version("1.0") + .set_env("test_env") + .set_service("test"); + let exporter = builder.build::().expect("build"); + let mut span = create_test_json_span(1234, 12342, 12341, 1, false); + span["name"] = json!("pb_span"); + let data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); + exporter.send(data.as_ref()).expect("send ok"); + }) + .await; + assert!(task_result.is_ok()); + assert_eq!(mock.calls_async().await, 1); + + // Decode the most recent request body as protobuf to prove wire correctness. + let received = mock.received_requests_async().await.unwrap(); + let body = &received[0].body; + let req = ExportTraceServiceRequest::decode(body.as_slice()).expect("valid protobuf"); + let svc = req.resource_spans[0] + .resource + .as_ref() + .unwrap() + .attributes + .iter() + .find(|kv| kv.key == "service.name") + .unwrap(); + use libdd_trace_protobuf::opentelemetry::proto::common::v1::any_value::Value; + assert!(matches!(svc.value.as_ref().unwrap().value, Some(Value::StringValue(ref s)) if s == "test")); + mock.delete(); + } +} +``` + +> If `received_requests_async` / body access differs in the pinned httpmock version, mirror the body-capture approach already used elsewhere in `libdd-data-pipeline/tests/`. Confirm `set_otlp_protocol` exists on the builder (Task 7) before running — order Task 7 before this step if executing strictly sequentially. + +- [ ] **Step 4: Run the new test (after Task 7's builder method exists)** + +Run: `cargo nextest run -p libdd-data-pipeline otlp_protobuf` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add libdd-data-pipeline/src/trace_exporter/mod.rs libdd-data-pipeline/tests/test_trace_exporter_otlp_protobuf_export.rs +git commit -m "feat(data-pipeline): dispatch OTLP encoder by protocol + protobuf test" +``` + +### Task 7: Builder `set_otlp_protocol` + +**Files:** +- Modify: `libdd-data-pipeline/src/trace_exporter/builder.rs` + +- [ ] **Step 1: Add the builder field + setter + use it in `build`** + +In `libdd-data-pipeline/src/trace_exporter/builder.rs`: +- add a field `otlp_protocol: OtlpProtocol` (defaults to `OtlpProtocol::default()` = `HttpJson`) to the builder struct and its `Default`/initialization; +- add the setter near `set_otlp_endpoint`: + +```rust + /// Selects the OTLP export protocol. Accepts `OtlpProtocol::HttpJson` (default) or + /// `OtlpProtocol::HttpProtobuf`. The host language resolves this from + /// `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` / `OTEL_EXPORTER_OTLP_PROTOCOL`. + pub fn set_otlp_protocol(&mut self, protocol: OtlpProtocol) -> &mut Self { + self.otlp_protocol = protocol; + self + } +``` +- in the `OtlpTraceConfig { ... }` construction, replace `protocol: OtlpProtocol::HttpJson` with `protocol: self.otlp_protocol`. + +- [ ] **Step 2: Verify it compiles** + +Run: `cargo check -p libdd-data-pipeline` +Expected: success. + +- [ ] **Step 3: Run Task 6's protobuf integration test now that the setter exists** + +Run: `cargo nextest run -p libdd-data-pipeline otlp` +Expected: PASS (both JSON and protobuf OTLP tests). + +- [ ] **Step 4: Commit** + +```bash +git add libdd-data-pipeline/src/trace_exporter/builder.rs +git commit -m "feat(data-pipeline): add TraceExporterBuilder::set_otlp_protocol" +``` + +--- + +## Phase 4 — C FFI (`libdd-data-pipeline-ffi`) + +### Task 8: `ddog_trace_exporter_config_set_otlp_protocol` + +**Files:** +- Modify: `libdd-data-pipeline-ffi/src/trace_exporter.rs` + +- [ ] **Step 1: Add the config field** + +In the `TraceExporterConfig` FFI struct (near the `otlp_endpoint: Option` field, ~line 85), add: +```rust + otlp_protocol: Option, +``` + +- [ ] **Step 2: Add the setter, modeled on `ddog_trace_exporter_config_set_otlp_endpoint`** + +After the existing OTLP endpoint setter (~line 499): + +```rust +/// Sets the OTLP export protocol. Accepts the OTel-standard values `http/json` (default) or +/// `http/protobuf`. `grpc` is rejected as not yet supported. The host language is responsible for +/// resolving the value (e.g. `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`). +#[no_mangle] +pub unsafe extern "C" fn ddog_trace_exporter_config_set_otlp_protocol( + config: Option<&mut TraceExporterConfig>, + protocol: CharSlice, +) -> Option> { + catch_panic!( + if let Some(handle) = config { + let value = match sanitize_string(protocol) { + Ok(s) => s, + Err(e) => return Some(e), + }; + match value.as_str() { + "http/json" | "http/protobuf" => { + handle.otlp_protocol = Some(value); + None + } + _ => gen_error!(ErrorCode::InvalidArgument), + } + } else { + gen_error!(ErrorCode::InvalidArgument) + }, + gen_error!(ErrorCode::Panic) + ) +} +``` + +- [ ] **Step 3: Apply the protocol in the exporter create function** + +Where the create fn calls `builder.set_otlp_endpoint(url)` (~line 566), add: +```rust + if let Some(ref proto) = config.otlp_protocol { + if let Ok(p) = proto.parse::() { + builder.set_otlp_protocol(p); + } + } +``` + +- [ ] **Step 4: Verify it compiles** + +Run: `cargo check -p libdd-data-pipeline-ffi` +Expected: success. (Confirm `OtlpProtocol` is re-exported from `libdd_data_pipeline::otlp::config`; if the ffi crate has a narrower re-export, use that path.) + +- [ ] **Step 5: Regenerate the C header** + +Run: +```bash +cargo build -p libdd-data-pipeline-ffi +``` +Then regenerate headers if the repo uses a header build step (check `builder`/`tools`); otherwise confirm the cbindgen-driven header includes `ddog_trace_exporter_config_set_otlp_protocol`. + +- [ ] **Step 6: Commit** + +```bash +git add libdd-data-pipeline-ffi/src/trace_exporter.rs +git commit -m "feat(data-pipeline-ffi): add ddog_trace_exporter_config_set_otlp_protocol" +``` + +--- + +## Phase 5 — libdatadog validation + PR + +### Task 9: Full validation gauntlet + +**Files:** none (validation). + +- [ ] **Step 1: Format** + +Run: `cargo +nightly-2026-02-08 fmt --all -- --check` +Expected: no diff. If it fails, run without `--check` and re-commit. + +- [ ] **Step 2: Clippy** + +Run: `cargo +stable clippy --workspace --all-targets --all-features -- -D warnings` +Expected: no warnings. + +- [ ] **Step 3: Tests (nextest + doc)** + +Run: +```bash +cargo nextest run --workspace --no-fail-fast +cargo nextest run --workspace --all-features --exclude builder --exclude test_spawn_from_lib +cargo test --doc +``` +Expected: all pass. (If `tracing_integration_tests::` need Docker, run `-E '!test(tracing_integration_tests::)'` and note it.) + +- [ ] **Step 4: FFI examples** + +Run: `cargo ffi-test` +Expected: C/C++ examples build + run. + +- [ ] **Step 5: License CSV (if Cargo.lock changed)** + +Run: +```bash +git diff --name-only origin/main -- Cargo.lock +``` +If `Cargo.lock` is listed: +```bash +./scripts/update_license_3rdparty.sh +cargo deny check +git add Cargo.lock LICENSE-3rdparty.csv +git commit -m "chore: update 3rd-party license CSV" +``` +Expected: `cargo deny check` clean. (Likely no Cargo.lock change since no new external crates were added.) + +- [ ] **Step 6: Apache headers on new files** + +Run: `./scripts/reformat_copyright.sh` then `git status`. +Expected: new `.rs` files carry the Apache header; commit any fixes. + +### Task 10: Open the libdatadog PR + +- [ ] **Step 1: Pre-push review (mandatory)** + +Invoke the `/pre-push-review` skill on the diff. + +- [ ] **Step 2: Push the branch** + +```bash +git push -u origin brian.marks/otlp-http-protobuf-export +``` + +- [ ] **Step 3: Create the draft PR with the repo template** + +Read `.github/pull_request_template.md`, fill all sections, and: +```bash +gh pr create --draft --label "AI Generated" --title "feat(data-pipeline): OTLP HTTP/protobuf trace export" --body-file +``` + +- [ ] **Step 4: Babysit CI** + +Invoke `/dd:pr-babysit` until CI is green (excluding `devflow/mergegate`). + +--- + +## Phase 6 — dd-trace-py wiring + local E2E (Tier 1) + +### Task 11: Set up a dd-trace-py worktree pointed at local libdatadog + +**Files:** +- Create: `/src/native/.cargo/config.toml` + +- [ ] **Step 1: Create a dd-trace-py worktree on a feature branch** + +```bash +cd /Users/brian.marks/dd/dd-trace-py +git fetch origin && git checkout main && git pull origin main +git worktree add ../dd-trace-py-otlp-protobuf -b brian.marks/otlp-http-protobuf-export +``` + +- [ ] **Step 2: Add the git-keyed cargo patch (NOT crates-io)** + +Create `../dd-trace-py-otlp-protobuf/src/native/.cargo/config.toml`: +```toml +[patch."https://github.com/DataDog/libdatadog"] +libdd-data-pipeline = { path = "/Users/brian.marks/go/src/github.com/DataDog/libdatadog-otlp-http-protobuf-export/libdd-data-pipeline" } +libdd-trace-utils = { path = "/Users/brian.marks/go/src/github.com/DataDog/libdatadog-otlp-http-protobuf-export/libdd-trace-utils" } +libdd-trace-protobuf = { path = "/Users/brian.marks/go/src/github.com/DataDog/libdatadog-otlp-http-protobuf-export/libdd-trace-protobuf" } +``` +> Add a patch line for every libdatadog crate in the modified set. If `cargo` reports an unpatched/duplicated source, add the named crate it points at. + +- [ ] **Step 3: Confirm the patch resolves** + +```bash +cd ../dd-trace-py-otlp-protobuf/src/native && cargo metadata --format-version 1 >/dev/null && echo OK +``` +Expected: `OK` (patch sources resolve). + +### Task 12: PyO3 `set_otlp_protocol` binding + +**Files:** +- Modify: `/src/native/data_pipeline/mod.rs` (after `set_otlp_headers`, ~line 189) + +- [ ] **Step 1: Add the binding, modeled on `set_otlp_endpoint`** + +```rust + fn set_otlp_protocol(mut slf: PyRefMut<'_, Self>, protocol: &'_ str) -> PyResult> { + slf.try_as_mut()?.set_otlp_protocol( + protocol + .parse() + .map_err(|e: String| pyo3::exceptions::PyValueError::new_err(e))?, + ); + Ok(slf.into()) + } +``` +> Import the builder's `OtlpProtocol` if the `.parse()` turbofish needs it: `use libdd_data_pipeline::otlp::config::OtlpProtocol;`. Match the exact `try_as_mut()` accessor used by the neighboring setters. + +- [ ] **Step 2: Build the native extension** + +```bash +cd /Users/brian.marks/dd/dd-trace-py-otlp-protobuf +python -m venv .venv && . .venv/bin/activate +pip install -e . 2>&1 | tail -20 +``` +Expected: build succeeds against the patched local libdatadog. + +- [ ] **Step 3: Smoke-test the binding from Python** + +```bash +python -c "from ddtrace.internal.native import TraceExporterBuilder as B; b=B(); b.set_otlp_protocol('http/protobuf'); print('ok')" +``` +Expected: `ok` (no exception). A bad value should raise `ValueError`. + +- [ ] **Step 4: Commit (dd-trace-py)** + +```bash +cd /Users/brian.marks/dd/dd-trace-py-otlp-protobuf +git add src/native/data_pipeline/mod.rs +git commit -m "feat(native): expose set_otlp_protocol on TraceExporterBuilder" +``` + +### Task 13: Wire `TRACES_PROTOCOL` through the writer + +**Files:** +- Modify: `/ddtrace/internal/writer/writer.py` (`_create_exporter`, ~line 827) +- Modify: `/ddtrace/internal/settings/_opentelemetry.py` (comments) + +- [ ] **Step 1: Pass the protocol when OTLP is enabled** + +In `_create_exporter`, after `builder.set_otlp_endpoint(self._otlp_endpoint)`: +```python + builder.set_otlp_protocol(otel_config.exporter.TRACES_PROTOCOL) +``` + +- [ ] **Step 2: Un-stub the comments in `_opentelemetry.py`** + +Remove the "TRACES_PROTOCOL is collected for telemetry but not yet used to switch transport" comment and update the `_derive_traces_endpoint` "libdatadog currently only supports http/json" note to reflect protobuf support. + +- [ ] **Step 3: Rebuild + commit** + +```bash +pip install -e . 2>&1 | tail -5 +git add ddtrace/internal/writer/writer.py ddtrace/internal/settings/_opentelemetry.py +git commit -m "feat(otlp): pass OTEL_EXPORTER_OTLP_TRACES_PROTOCOL to the native exporter" +``` + +### Task 14: Local protobuf-decoding receiver E2E + +**Files:** +- Create (scratch, not committed): `/tmp/otlp_recv.py`, `/tmp/otlp_app.py` + +- [ ] **Step 1: Write the receiver** + +`/tmp/otlp_recv.py`: +```python +from http.server import BaseHTTPRequestHandler, HTTPServer +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + +class H(BaseHTTPRequestHandler): + def do_POST(self): + n = int(self.headers.get("content-length", 0)) + body = self.rfile.read(n) + ct = self.headers.get("content-type", "") + assert ct == "application/x-protobuf", f"bad content-type: {ct}" + req = ExportTraceServiceRequest() + req.ParseFromString(body) # raises on malformed protobuf + span = req.resource_spans[0].scope_spans[0].spans[0] + print("OK decoded:", span.name, "trace_id_len", len(span.trace_id)) + self.send_response(200); self.end_headers(); self.wfile.write(b"") + +HTTPServer(("127.0.0.1", 4318), H).serve_forever() +``` +Install the proto package in the venv: `pip install opentelemetry-proto`. + +- [ ] **Step 2: Write the instrumented app** + +`/tmp/otlp_app.py`: +```python +from ddtrace import tracer +with tracer.trace("e2e_protobuf_span", resource="GET /e2e"): + pass +tracer.flush() +``` + +- [ ] **Step 3: Run protobuf E2E** + +```bash +python /tmp/otlp_recv.py & # terminal 1 +OTEL_TRACES_EXPORTER=otlp \ +OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf \ +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://127.0.0.1:4318/v1/traces \ +python /tmp/otlp_app.py +``` +Expected: receiver prints `OK decoded: GET /e2e trace_id_len 16`. Ensure `DD_TRACE_AGENT_PROTOCOL_VERSION` is unset. + +- [ ] **Step 4: Run JSON regression E2E** + +Re-run with `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/json` and a JSON-aware receiver variant (assert `content-type: application/json`, `json.loads(body)`). +Expected: JSON path still works unchanged. + +--- + +## Phase 7 — system-tests (Tier 2) + +### Task 15: Run system-tests OTLP scenario against local builds + +**Files:** none (uses `apm-ecosystems:system-tests-local`). + +- [ ] **Step 1: Identify the OTLP trace-export scenario** + +Invoke `apm-ecosystems:system-tests-local`. In the system-tests checkout, locate the scenario(s) covering OTLP trace export / `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` for Python. Record the scenario name(s). +> This is the one item the spec left open. Resolve it here before running. + +- [ ] **Step 2: Build system-tests against the local dd-trace-py (which is built against local libdatadog)** + +Follow the skill's flow to point system-tests at the `dd-trace-py-otlp-protobuf` build. + +- [ ] **Step 3: Run the OTLP scenario with `http/protobuf`** + +Run the identified scenario; assert it passes with protocol set to `http/protobuf`. Capture output. + +- [ ] **Step 4: Record results** + +Note pass/fail and any scenario gaps in the dd-trace-py PR description. + +--- + +## Phase 8 — sdk-backend-verify (Tier 3) + +### Task 16: Full-chain backend verification + +**Files:** none (uses `apm-ecosystems:sdk-backend-verify` + the backend-integrated flow in CLAUDE.md). + +- [ ] **Step 1: Start an OTLP-capable receiver that forwards to the backend** + +Either the DD Agent with OTLP intake enabled on `:4318`, or the OTel Collector with a Datadog exporter. Use the local agent setup from CLAUDE.md (test-org API key from 1Password). Use a unique `DD_SERVICE` per run to avoid the RC/classification cache. + +- [ ] **Step 2: Emit protobuf OTLP traffic** + +```bash +OTEL_TRACES_EXPORTER=otlp \ +OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf \ +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://127.0.0.1:4318/v1/traces \ +DD_SERVICE=bm-otlp-pb-$(date +%H%M) \ +python /tmp/otlp_app.py +``` + +- [ ] **Step 3: Verify in the backend** + +Invoke `apm-ecosystems:sdk-backend-verify` (or the spans search/aggregate APIs in CLAUDE.md) to confirm the spans landed with correct service, resource, and a 128-bit trace_id. Capture the evidence. + +- [ ] **Step 4: Record results in the dd-trace-py PR** + +--- + +## Phase 9 — dd-trace-py PR + +### Task 17: Open the dd-trace-py PR (depends on a libdatadog release) + +- [ ] **Step 1: Add the cargo dependency bump note** + +The `src/native/Cargo.toml` git pins stay at the current `rev` until libdatadog ships a release containing Phase 1–4. Document in the PR that the rev bump + removal of the local `.cargo/config.toml` patch is required before merge. Do not commit the local `.cargo/config.toml` patch. + +- [ ] **Step 2: Pre-push review + push** + +Invoke `/pre-push-review`, then push `brian.marks/otlp-http-protobuf-export`. + +- [ ] **Step 3: Create the draft PR with the repo template** + +Read dd-trace-py's PR template, fill it (including the Tier 1–3 validation evidence), and: +```bash +gh pr create --draft --label "AI Generated" --title "feat(otlp): select OTLP trace protocol (http/json|http/protobuf)" --body-file +``` + +- [ ] **Step 4: Babysit CI** + +Invoke `/dd:pr-babysit`. + +--- + +## Self-review notes (plan vs spec) + +- **Spec coverage:** type vendoring (Task 1), serde→prost converter (Task 2), encoders (Task 3), protocol `FromStr` (Task 4), content-type (Task 5), dispatch (Task 6), builder (Task 7), FFI (Task 8), validation gauntlet (Task 9), libdatadog PR (Task 10), dd-trace-py PyO3 + writer (Tasks 12–13), local E2E (Task 14), system-tests (Task 15), sdk-backend-verify (Task 16), dd-trace-py PR (Task 17). All spec sections covered. +- **Deviation:** dropped the `otlp-protobuf` cargo feature gate (justified in the header — types are unconditionally compiled via vendoring; YAGNI). +- **Known-unknown resolved in plan:** the system-tests scenario name is resolved in Task 15 Step 1 rather than left as a spec TODO. +- **Type consistency:** `OtlpProtocol` (config.rs) used consistently across Tasks 4/6/7/8/12; `encode_otlp_json`/`encode_otlp_protobuf` defined in Task 3 and used in Task 6; `ExportTraceServiceRequest` (serde) vs prost `ExportTraceServiceRequest` disambiguated via aliases. +- **Open verification points flagged inline:** exact generated prost field names (Task 2 Step 3 note), httpmock body-capture API (Task 6 Step 3 note), PyO3 `try_as_mut` accessor (Task 12 Step 1 note). From 07c729664f1950d0a98036c4450b600d95eddc58 Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 13:50:44 -0400 Subject: [PATCH 03/17] feat(trace-protobuf): vendor + generate OTLP trace/collector prost types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vendors opentelemetry/proto/trace/v1/trace.proto and opentelemetry/proto/collector/trace/v1/trace_service.proto from open-telemetry/opentelemetry-proto commit 1e725b853bc8f6b46ee62e8232e4c83017b9536f (matching the already-vendored common.proto and resource.proto). Adds both protos to the prost_build compile list in build.rs, generates the committed Rust types (opentelemetry.proto.trace.v1.rs and opentelemetry.proto.collector.trace.v1.rs), and updates _includes.rs. Also qualifies "Span" → "pb.Span" / "pb.idx.Span" in build.rs type_attribute calls to prevent serde derives from leaking into the new opentelemetry::proto::trace::v1::Span type. --- libdd-trace-protobuf/build.rs | 19 +- libdd-trace-protobuf/src/_includes.rs | 12 + .../opentelemetry.proto.collector.trace.v1.rs | 54 +++ .../src/opentelemetry.proto.trace.v1.rs | 438 ++++++++++++++++++ .../collector/trace/v1/trace_service.proto | 80 ++++ .../opentelemetry/proto/trace/v1/trace.proto | 362 +++++++++++++++ 6 files changed, 963 insertions(+), 2 deletions(-) create mode 100644 libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs create mode 100644 libdd-trace-protobuf/src/opentelemetry.proto.trace.v1.rs create mode 100644 libdd-trace-protobuf/src/pb/opentelemetry/proto/collector/trace/v1/trace_service.proto create mode 100644 libdd-trace-protobuf/src/pb/opentelemetry/proto/trace/v1/trace.proto diff --git a/libdd-trace-protobuf/build.rs b/libdd-trace-protobuf/build.rs index c9c891a681..ba99a075a9 100644 --- a/libdd-trace-protobuf/build.rs +++ b/libdd-trace-protobuf/build.rs @@ -62,9 +62,14 @@ fn generate_protobuf() { config.field_attribute(".pb.SpanLink.tracestate", "#[serde(default)]"); config.field_attribute(".pb.SpanLink.flags", "#[serde(default)]"); - config.type_attribute("Span", "#[derive(Deserialize, Serialize)]"); + config.type_attribute("pb.Span", "#[derive(Deserialize, Serialize)]"); config.type_attribute( - "Span", + "pb.Span", + r#"#[cfg_attr(feature = "fuzzing", derive(bolero::TypeGenerator))]"#, + ); + config.type_attribute("pb.idx.Span", "#[derive(Deserialize, Serialize)]"); + config.type_attribute( + "pb.idx.Span", r#"#[cfg_attr(feature = "fuzzing", derive(bolero::TypeGenerator))]"#, ); config.field_attribute( @@ -319,6 +324,8 @@ fn generate_protobuf() { "src/pb/stats.proto", "src/pb/remoteconfig.proto", "src/pb/opentelemetry/proto/common/v1/process_context.proto", + "src/pb/opentelemetry/proto/trace/v1/trace.proto", + "src/pb/opentelemetry/proto/collector/trace/v1/trace_service.proto", "src/pb/idx/tracer_payload.proto", "src/pb/idx/span.proto", ], @@ -363,6 +370,14 @@ fn generate_protobuf() { otel_license, &output_path.join("opentelemetry.proto.common.v1.rs"), ); + prepend_to_file( + otel_license, + &output_path.join("opentelemetry.proto.trace.v1.rs"), + ); + prepend_to_file( + otel_license, + &output_path.join("opentelemetry.proto.collector.trace.v1.rs"), + ); } #[cfg(feature = "generate-protobuf")] diff --git a/libdd-trace-protobuf/src/_includes.rs b/libdd-trace-protobuf/src/_includes.rs index 1628f52c39..0377bbe9dc 100644 --- a/libdd-trace-protobuf/src/_includes.rs +++ b/libdd-trace-protobuf/src/_includes.rs @@ -4,6 +4,13 @@ // This file is @generated by prost-build. pub mod opentelemetry { pub mod proto { + pub mod collector { + pub mod trace { + pub mod v1 { + include!("opentelemetry.proto.collector.trace.v1.rs"); + } + } + } pub mod common { pub mod v1 { include!("opentelemetry.proto.common.v1.rs"); @@ -14,6 +21,11 @@ pub mod opentelemetry { include!("opentelemetry.proto.resource.v1.rs"); } } + pub mod trace { + pub mod v1 { + include!("opentelemetry.proto.trace.v1.rs"); + } + } } } pub mod pb { diff --git a/libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs b/libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs new file mode 100644 index 0000000000..3a1e3db44d --- /dev/null +++ b/libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs @@ -0,0 +1,54 @@ +// Copyright 2019, OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// This file is @generated by prost-build. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ExportTraceServiceRequest { + /// An array of ResourceSpans. + /// For data coming from a single resource this array will typically contain one + /// element. Intermediary nodes (such as OpenTelemetry Collector) that receive + /// data from multiple origins typically batch the data before forwarding further and + /// in that case this array will contain multiple elements. + #[prost(message, repeated, tag = "1")] + pub resource_spans: ::prost::alloc::vec::Vec< + super::super::super::trace::v1::ResourceSpans, + >, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ExportTraceServiceResponse { + /// The details of a partially successful export request. + /// + /// If the request is only partially accepted + /// (i.e. when the server accepts only parts of the data and rejects the rest) + /// the server MUST initialize the `partial_success` field and MUST + /// set the `rejected_` with the number of items it rejected. + /// + /// Servers MAY also make use of the `partial_success` field to convey + /// warnings/suggestions to senders even when the request was fully accepted. + /// In such cases, the `rejected_` MUST have a value of `0` and + /// the `error_message` MUST be non-empty. + /// + /// A `partial_success` message with an empty value (rejected_ = 0 and + /// `error_message` = "") is equivalent to it not being set/present. Senders + /// SHOULD interpret it the same way as in the full success case. + #[prost(message, optional, tag = "1")] + pub partial_success: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ExportTracePartialSuccess { + /// The number of rejected spans. + /// + /// A `rejected_` field holding a `0` value indicates that the + /// request was fully accepted. + #[prost(int64, tag = "1")] + pub rejected_spans: i64, + /// A developer-facing human-readable message in English. It should be used + /// either to explain why the server rejected parts of the data during a partial + /// success or to convey warnings/suggestions during a full success. The message + /// should offer guidance on how users can address such issues. + /// + /// error_message is an optional field. An error_message with an empty value + /// is equivalent to it not being set. + #[prost(string, tag = "2")] + pub error_message: ::prost::alloc::string::String, +} diff --git a/libdd-trace-protobuf/src/opentelemetry.proto.trace.v1.rs b/libdd-trace-protobuf/src/opentelemetry.proto.trace.v1.rs new file mode 100644 index 0000000000..2e6b3ec3a7 --- /dev/null +++ b/libdd-trace-protobuf/src/opentelemetry.proto.trace.v1.rs @@ -0,0 +1,438 @@ +// Copyright 2019, OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// This file is @generated by prost-build. +/// TracesData represents the traces data that can be stored in a persistent storage, +/// OR can be embedded by other protocols that transfer OTLP traces data but do +/// not implement the OTLP protocol. +/// +/// The main difference between this message and collector protocol is that +/// in this message there will not be any "control" or "metadata" specific to +/// OTLP protocol. +/// +/// When new fields are added into this message, the OTLP request MUST be updated +/// as well. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TracesData { + /// An array of ResourceSpans. + /// For data coming from a single resource this array will typically contain + /// one element. Intermediary nodes that receive data from multiple origins + /// typically batch the data before forwarding further and in that case this + /// array will contain multiple elements. + #[prost(message, repeated, tag = "1")] + pub resource_spans: ::prost::alloc::vec::Vec, +} +/// A collection of ScopeSpans from a Resource. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ResourceSpans { + /// The resource for the spans in this message. + /// If this field is not set then no resource info is known. + #[prost(message, optional, tag = "1")] + pub resource: ::core::option::Option, + /// A list of ScopeSpans that originate from a resource. + #[prost(message, repeated, tag = "2")] + pub scope_spans: ::prost::alloc::vec::Vec, + /// The Schema URL, if known. This is the identifier of the Schema that the resource data + /// is recorded in. Notably, the last part of the URL path is the version number of the + /// schema: http\[s\]://server\[:port\]/path/. To learn more about Schema URL see + /// + /// This schema_url applies to the data in the "resource" field. It does not apply + /// to the data in the "scope_spans" field which have their own schema_url field. + #[prost(string, tag = "3")] + pub schema_url: ::prost::alloc::string::String, +} +/// A collection of Spans produced by an InstrumentationScope. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ScopeSpans { + /// The instrumentation scope information for the spans in this message. + /// Semantically when InstrumentationScope isn't set, it is equivalent with + /// an empty instrumentation scope name (unknown). + #[prost(message, optional, tag = "1")] + pub scope: ::core::option::Option, + /// A list of Spans that originate from an instrumentation scope. + #[prost(message, repeated, tag = "2")] + pub spans: ::prost::alloc::vec::Vec, + /// The Schema URL, if known. This is the identifier of the Schema that the span data + /// is recorded in. Notably, the last part of the URL path is the version number of the + /// schema: http\[s\]://server\[:port\]/path/. To learn more about Schema URL see + /// + /// This schema_url applies to the data in the "scope" field and all spans and span + /// events in the "spans" field. + #[prost(string, tag = "3")] + pub schema_url: ::prost::alloc::string::String, +} +/// A Span represents a single operation performed by a single component of the system. +/// +/// The next available field id is 17. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Span { + /// A unique identifier for a trace. All spans from the same trace share + /// the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR + /// of length other than 16 bytes is considered invalid (empty string in OTLP/JSON + /// is zero-length and thus is also invalid). + /// + /// This field is required. + #[prost(bytes = "vec", tag = "1")] + pub trace_id: ::prost::alloc::vec::Vec, + /// A unique identifier for a span within a trace, assigned when the span + /// is created. The ID is an 8-byte array. An ID with all zeroes OR of length + /// other than 8 bytes is considered invalid (empty string in OTLP/JSON + /// is zero-length and thus is also invalid). + /// + /// This field is required. + #[prost(bytes = "vec", tag = "2")] + pub span_id: ::prost::alloc::vec::Vec, + /// trace_state conveys information about request position in multiple distributed tracing graphs. + /// It is a trace_state in w3c-trace-context format: + /// See also for more details about this field. + #[prost(string, tag = "3")] + pub trace_state: ::prost::alloc::string::String, + /// The `span_id` of this span's parent span. If this is a root span, then this + /// field must be empty. The ID is an 8-byte array. + #[prost(bytes = "vec", tag = "4")] + pub parent_span_id: ::prost::alloc::vec::Vec, + /// Flags, a bit field. + /// + /// Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + /// Context specification. To read the 8-bit W3C trace flag, use + /// `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + /// + /// See for the flag definitions. + /// + /// Bits 8 and 9 represent the 3 states of whether a span's parent + /// is remote. The states are (unknown, is not remote, is remote). + /// To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + /// To read whether the span is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + /// + /// When creating span messages, if the message is logically forwarded from another source + /// with an equivalent flags fields (i.e., usually another OTLP span message), the field SHOULD + /// be copied as-is. If creating from a source that does not have an equivalent flags field + /// (such as a runtime representation of an OpenTelemetry span), the high 22 bits MUST + /// be set to zero. + /// Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + /// + /// \[Optional\]. + #[prost(fixed32, tag = "16")] + pub flags: u32, + /// A description of the span's operation. + /// + /// For example, the name can be a qualified method name or a file name + /// and a line number where the operation is called. A best practice is to use + /// the same display name at the same call point in an application. + /// This makes it easier to correlate spans in different traces. + /// + /// This field is semantically required to be set to non-empty string. + /// Empty value is equivalent to an unknown span name. + /// + /// This field is required. + #[prost(string, tag = "5")] + pub name: ::prost::alloc::string::String, + /// Distinguishes between spans generated in a particular context. For example, + /// two spans with the same name may be distinguished using `CLIENT` (caller) + /// and `SERVER` (callee) to identify queueing latency associated with the span. + #[prost(enumeration = "span::SpanKind", tag = "6")] + pub kind: i32, + /// The start time of the span. On the client side, this is the time + /// kept by the local machine where the span execution starts. On the server side, this + /// is the time when the server's application handler starts running. + /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + /// + /// This field is semantically required and it is expected that end_time >= start_time. + #[prost(fixed64, tag = "7")] + pub start_time_unix_nano: u64, + /// The end time of the span. On the client side, this is the time + /// kept by the local machine where the span execution ends. On the server side, this + /// is the time when the server application handler stops running. + /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + /// + /// This field is semantically required and it is expected that end_time >= start_time. + #[prost(fixed64, tag = "8")] + pub end_time_unix_nano: u64, + /// A collection of key/value pairs. Note, global attributes + /// like server name can be set using the resource API. Examples of attributes: + /// + /// "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + /// "/http/server_latency": 300 + /// "example.com/myattribute": true + /// "example.com/score": 10.239 + /// + /// Attribute keys MUST be unique (it is not allowed to have more than one + /// attribute with the same key). + /// The behavior of software that receives duplicated keys can be unpredictable. + #[prost(message, repeated, tag = "9")] + pub attributes: ::prost::alloc::vec::Vec, + /// The number of attributes that were discarded. Attributes + /// can be discarded because their keys are too long or because there are too many + /// attributes. If this value is 0, then no attributes were dropped. + #[prost(uint32, tag = "10")] + pub dropped_attributes_count: u32, + /// A collection of Event items. + #[prost(message, repeated, tag = "11")] + pub events: ::prost::alloc::vec::Vec, + /// The number of dropped events. If the value is 0, then no + /// events were dropped. + #[prost(uint32, tag = "12")] + pub dropped_events_count: u32, + /// A collection of Links, which are references from this span to a span + /// in the same or different trace. + #[prost(message, repeated, tag = "13")] + pub links: ::prost::alloc::vec::Vec, + /// The number of dropped links after the maximum size was + /// enforced. If this value is 0, then no links were dropped. + #[prost(uint32, tag = "14")] + pub dropped_links_count: u32, + /// An optional final status for this span. Semantically when Status isn't set, it means + /// span's status code is unset, i.e. assume STATUS_CODE_UNSET (code = 0). + #[prost(message, optional, tag = "15")] + pub status: ::core::option::Option, +} +/// Nested message and enum types in `Span`. +pub mod span { + /// Event is a time-stamped annotation of the span, consisting of user-supplied + /// text description and key-value pairs. + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Event { + /// The time the event occurred. + #[prost(fixed64, tag = "1")] + pub time_unix_nano: u64, + /// The name of the event. + /// This field is semantically required to be set to non-empty string. + #[prost(string, tag = "2")] + pub name: ::prost::alloc::string::String, + /// A collection of attribute key/value pairs on the event. + /// Attribute keys MUST be unique (it is not allowed to have more than one + /// attribute with the same key). + /// The behavior of software that receives duplicated keys can be unpredictable. + #[prost(message, repeated, tag = "3")] + pub attributes: ::prost::alloc::vec::Vec< + super::super::super::common::v1::KeyValue, + >, + /// The number of dropped attributes. If the value is 0, + /// then no attributes were dropped. + #[prost(uint32, tag = "4")] + pub dropped_attributes_count: u32, + } + /// A pointer from the current span to another span in the same trace or in a + /// different trace. For example, this can be used in batching operations, + /// where a single batch handler processes multiple requests from different + /// traces or when the handler receives a request from a different project. + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Link { + /// A unique identifier of a trace that this linked span is part of. The ID is a + /// 16-byte array. + #[prost(bytes = "vec", tag = "1")] + pub trace_id: ::prost::alloc::vec::Vec, + /// A unique identifier for the linked span. The ID is an 8-byte array. + #[prost(bytes = "vec", tag = "2")] + pub span_id: ::prost::alloc::vec::Vec, + /// The trace_state associated with the link. + #[prost(string, tag = "3")] + pub trace_state: ::prost::alloc::string::String, + /// A collection of attribute key/value pairs on the link. + /// Attribute keys MUST be unique (it is not allowed to have more than one + /// attribute with the same key). + /// The behavior of software that receives duplicated keys can be unpredictable. + #[prost(message, repeated, tag = "4")] + pub attributes: ::prost::alloc::vec::Vec< + super::super::super::common::v1::KeyValue, + >, + /// The number of dropped attributes. If the value is 0, + /// then no attributes were dropped. + #[prost(uint32, tag = "5")] + pub dropped_attributes_count: u32, + /// Flags, a bit field. + /// + /// Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + /// Context specification. To read the 8-bit W3C trace flag, use + /// `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + /// + /// See for the flag definitions. + /// + /// Bits 8 and 9 represent the 3 states of whether the link is remote. + /// The states are (unknown, is not remote, is remote). + /// To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + /// To read whether the link is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + /// + /// Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + /// When creating new spans, bits 10-31 (most-significant 22-bits) MUST be zero. + /// + /// \[Optional\]. + #[prost(fixed32, tag = "6")] + pub flags: u32, + } + /// SpanKind is the type of span. Can be used to specify additional relationships between spans + /// in addition to a parent/child relationship. + #[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + ::prost::Enumeration + )] + #[repr(i32)] + pub enum SpanKind { + /// Unspecified. Do NOT use as default. + /// Implementations MAY assume SpanKind to be INTERNAL when receiving UNSPECIFIED. + Unspecified = 0, + /// Indicates that the span represents an internal operation within an application, + /// as opposed to an operation happening at the boundaries. Default value. + Internal = 1, + /// Indicates that the span covers server-side handling of an RPC or other + /// remote network request. + Server = 2, + /// Indicates that the span describes a request to some remote service. + Client = 3, + /// Indicates that the span describes a producer sending a message to a broker. + /// Unlike CLIENT and SERVER, there is often no direct critical path latency relationship + /// between producer and consumer spans. A PRODUCER span ends when the message was accepted + /// by the broker while the logical processing of the message might span a much longer time. + Producer = 4, + /// Indicates that the span describes consumer receiving a message from a broker. + /// Like the PRODUCER kind, there is often no direct critical path latency relationship + /// between producer and consumer spans. + Consumer = 5, + } + impl SpanKind { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "SPAN_KIND_UNSPECIFIED", + Self::Internal => "SPAN_KIND_INTERNAL", + Self::Server => "SPAN_KIND_SERVER", + Self::Client => "SPAN_KIND_CLIENT", + Self::Producer => "SPAN_KIND_PRODUCER", + Self::Consumer => "SPAN_KIND_CONSUMER", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "SPAN_KIND_UNSPECIFIED" => Some(Self::Unspecified), + "SPAN_KIND_INTERNAL" => Some(Self::Internal), + "SPAN_KIND_SERVER" => Some(Self::Server), + "SPAN_KIND_CLIENT" => Some(Self::Client), + "SPAN_KIND_PRODUCER" => Some(Self::Producer), + "SPAN_KIND_CONSUMER" => Some(Self::Consumer), + _ => None, + } + } + } +} +/// The Status type defines a logical error model that is suitable for different +/// programming environments, including REST APIs and RPC APIs. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Status { + /// A developer-facing human readable error message. + #[prost(string, tag = "2")] + pub message: ::prost::alloc::string::String, + /// The status code. + #[prost(enumeration = "status::StatusCode", tag = "3")] + pub code: i32, +} +/// Nested message and enum types in `Status`. +pub mod status { + /// For the semantics of status codes see + /// + #[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + ::prost::Enumeration + )] + #[repr(i32)] + pub enum StatusCode { + /// The default status. + Unset = 0, + /// The Span has been validated by an Application developer or Operator to + /// have completed successfully. + Ok = 1, + /// The Span contains an error. + Error = 2, + } + impl StatusCode { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unset => "STATUS_CODE_UNSET", + Self::Ok => "STATUS_CODE_OK", + Self::Error => "STATUS_CODE_ERROR", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "STATUS_CODE_UNSET" => Some(Self::Unset), + "STATUS_CODE_OK" => Some(Self::Ok), + "STATUS_CODE_ERROR" => Some(Self::Error), + _ => None, + } + } + } +} +/// SpanFlags represents constants used to interpret the +/// Span.flags field, which is protobuf 'fixed32' type and is to +/// be used as bit-fields. Each non-zero value defined in this enum is +/// a bit-mask. To extract the bit-field, for example, use an +/// expression like: +/// +/// (span.flags & SPAN_FLAGS_TRACE_FLAGS_MASK) +/// +/// See for the flag definitions. +/// +/// Note that Span flags were introduced in version 1.1 of the +/// OpenTelemetry protocol. Older Span producers do not set this +/// field, consequently consumers should not rely on the absence of a +/// particular flag bit to indicate the presence of a particular feature. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum SpanFlags { + /// The zero value for the enum. Should not be used for comparisons. + /// Instead use bitwise "and" with the appropriate mask as shown above. + DoNotUse = 0, + /// Bits 0-7 are used for trace flags. + TraceFlagsMask = 255, + /// Bits 8 and 9 are used to indicate that the parent span or link span is remote. + /// Bit 8 (`HAS_IS_REMOTE`) indicates whether the value is known. + /// Bit 9 (`IS_REMOTE`) indicates whether the span or link is remote. + ContextHasIsRemoteMask = 256, + ContextIsRemoteMask = 512, +} +impl SpanFlags { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::DoNotUse => "SPAN_FLAGS_DO_NOT_USE", + Self::TraceFlagsMask => "SPAN_FLAGS_TRACE_FLAGS_MASK", + Self::ContextHasIsRemoteMask => "SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK", + Self::ContextIsRemoteMask => "SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "SPAN_FLAGS_DO_NOT_USE" => Some(Self::DoNotUse), + "SPAN_FLAGS_TRACE_FLAGS_MASK" => Some(Self::TraceFlagsMask), + "SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK" => Some(Self::ContextHasIsRemoteMask), + "SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK" => Some(Self::ContextIsRemoteMask), + _ => None, + } + } +} diff --git a/libdd-trace-protobuf/src/pb/opentelemetry/proto/collector/trace/v1/trace_service.proto b/libdd-trace-protobuf/src/pb/opentelemetry/proto/collector/trace/v1/trace_service.proto new file mode 100644 index 0000000000..1e77256209 --- /dev/null +++ b/libdd-trace-protobuf/src/pb/opentelemetry/proto/collector/trace/v1/trace_service.proto @@ -0,0 +1,80 @@ +// This file was vendored from open-telemetry/opentelemetry-proto at commit +// 1e725b853bc8f6b46ee62e8232e4c83017b9536f. + +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.collector.trace.v1; + +import "opentelemetry/proto/trace/v1/trace.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Collector.Trace.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.collector.trace.v1"; +option java_outer_classname = "TraceServiceProto"; +option go_package = "go.opentelemetry.io/proto/otlp/collector/trace/v1"; + +// Service that can be used to push spans between one Application instrumented with +// OpenTelemetry and a collector, or between a collector and a central collector (in this +// case spans are sent/received to/from multiple Applications). +service TraceService { + rpc Export(ExportTraceServiceRequest) returns (ExportTraceServiceResponse) {} +} + +message ExportTraceServiceRequest { + // An array of ResourceSpans. + // For data coming from a single resource this array will typically contain one + // element. Intermediary nodes (such as OpenTelemetry Collector) that receive + // data from multiple origins typically batch the data before forwarding further and + // in that case this array will contain multiple elements. + repeated opentelemetry.proto.trace.v1.ResourceSpans resource_spans = 1; +} + +message ExportTraceServiceResponse { + // The details of a partially successful export request. + // + // If the request is only partially accepted + // (i.e. when the server accepts only parts of the data and rejects the rest) + // the server MUST initialize the `partial_success` field and MUST + // set the `rejected_` with the number of items it rejected. + // + // Servers MAY also make use of the `partial_success` field to convey + // warnings/suggestions to senders even when the request was fully accepted. + // In such cases, the `rejected_` MUST have a value of `0` and + // the `error_message` MUST be non-empty. + // + // A `partial_success` message with an empty value (rejected_ = 0 and + // `error_message` = "") is equivalent to it not being set/present. Senders + // SHOULD interpret it the same way as in the full success case. + ExportTracePartialSuccess partial_success = 1; +} + +message ExportTracePartialSuccess { + // The number of rejected spans. + // + // A `rejected_` field holding a `0` value indicates that the + // request was fully accepted. + int64 rejected_spans = 1; + + // A developer-facing human-readable message in English. It should be used + // either to explain why the server rejected parts of the data during a partial + // success or to convey warnings/suggestions during a full success. The message + // should offer guidance on how users can address such issues. + // + // error_message is an optional field. An error_message with an empty value + // is equivalent to it not being set. + string error_message = 2; +} \ No newline at end of file diff --git a/libdd-trace-protobuf/src/pb/opentelemetry/proto/trace/v1/trace.proto b/libdd-trace-protobuf/src/pb/opentelemetry/proto/trace/v1/trace.proto new file mode 100644 index 0000000000..69564c256a --- /dev/null +++ b/libdd-trace-protobuf/src/pb/opentelemetry/proto/trace/v1/trace.proto @@ -0,0 +1,362 @@ +// This file was vendored from open-telemetry/opentelemetry-proto at commit +// 1e725b853bc8f6b46ee62e8232e4c83017b9536f. + +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.trace.v1; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Trace.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.trace.v1"; +option java_outer_classname = "TraceProto"; +option go_package = "go.opentelemetry.io/proto/otlp/trace/v1"; + +// TracesData represents the traces data that can be stored in a persistent storage, +// OR can be embedded by other protocols that transfer OTLP traces data but do +// not implement the OTLP protocol. +// +// The main difference between this message and collector protocol is that +// in this message there will not be any "control" or "metadata" specific to +// OTLP protocol. +// +// When new fields are added into this message, the OTLP request MUST be updated +// as well. +message TracesData { + // An array of ResourceSpans. + // For data coming from a single resource this array will typically contain + // one element. Intermediary nodes that receive data from multiple origins + // typically batch the data before forwarding further and in that case this + // array will contain multiple elements. + repeated ResourceSpans resource_spans = 1; +} + +// A collection of ScopeSpans from a Resource. +message ResourceSpans { + reserved 1000; + + // The resource for the spans in this message. + // If this field is not set then no resource info is known. + opentelemetry.proto.resource.v1.Resource resource = 1; + + // A list of ScopeSpans that originate from a resource. + repeated ScopeSpans scope_spans = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "resource" field. It does not apply + // to the data in the "scope_spans" field which have their own schema_url field. + string schema_url = 3; +} + +// A collection of Spans produced by an InstrumentationScope. +message ScopeSpans { + // The instrumentation scope information for the spans in this message. + // Semantically when InstrumentationScope isn't set, it is equivalent with + // an empty instrumentation scope name (unknown). + opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + + // A list of Spans that originate from an instrumentation scope. + repeated Span spans = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the span data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "scope" field and all spans and span + // events in the "spans" field. + string schema_url = 3; +} + +// A Span represents a single operation performed by a single component of the system. +// +// The next available field id is 17. +message Span { + // A unique identifier for a trace. All spans from the same trace share + // the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR + // of length other than 16 bytes is considered invalid (empty string in OTLP/JSON + // is zero-length and thus is also invalid). + // + // This field is required. + bytes trace_id = 1; + + // A unique identifier for a span within a trace, assigned when the span + // is created. The ID is an 8-byte array. An ID with all zeroes OR of length + // other than 8 bytes is considered invalid (empty string in OTLP/JSON + // is zero-length and thus is also invalid). + // + // This field is required. + bytes span_id = 2; + + // trace_state conveys information about request position in multiple distributed tracing graphs. + // It is a trace_state in w3c-trace-context format: https://www.w3.org/TR/trace-context/#tracestate-header + // See also https://github.com/w3c/distributed-tracing for more details about this field. + string trace_state = 3; + + // The `span_id` of this span's parent span. If this is a root span, then this + // field must be empty. The ID is an 8-byte array. + bytes parent_span_id = 4; + + // Flags, a bit field. + // + // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + // Context specification. To read the 8-bit W3C trace flag, use + // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + // + // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. + // + // Bits 8 and 9 represent the 3 states of whether a span's parent + // is remote. The states are (unknown, is not remote, is remote). + // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + // To read whether the span is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + // + // When creating span messages, if the message is logically forwarded from another source + // with an equivalent flags fields (i.e., usually another OTLP span message), the field SHOULD + // be copied as-is. If creating from a source that does not have an equivalent flags field + // (such as a runtime representation of an OpenTelemetry span), the high 22 bits MUST + // be set to zero. + // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + // + // [Optional]. + fixed32 flags = 16; + + // A description of the span's operation. + // + // For example, the name can be a qualified method name or a file name + // and a line number where the operation is called. A best practice is to use + // the same display name at the same call point in an application. + // This makes it easier to correlate spans in different traces. + // + // This field is semantically required to be set to non-empty string. + // Empty value is equivalent to an unknown span name. + // + // This field is required. + string name = 5; + + // SpanKind is the type of span. Can be used to specify additional relationships between spans + // in addition to a parent/child relationship. + enum SpanKind { + // Unspecified. Do NOT use as default. + // Implementations MAY assume SpanKind to be INTERNAL when receiving UNSPECIFIED. + SPAN_KIND_UNSPECIFIED = 0; + + // Indicates that the span represents an internal operation within an application, + // as opposed to an operation happening at the boundaries. Default value. + SPAN_KIND_INTERNAL = 1; + + // Indicates that the span covers server-side handling of an RPC or other + // remote network request. + SPAN_KIND_SERVER = 2; + + // Indicates that the span describes a request to some remote service. + SPAN_KIND_CLIENT = 3; + + // Indicates that the span describes a producer sending a message to a broker. + // Unlike CLIENT and SERVER, there is often no direct critical path latency relationship + // between producer and consumer spans. A PRODUCER span ends when the message was accepted + // by the broker while the logical processing of the message might span a much longer time. + SPAN_KIND_PRODUCER = 4; + + // Indicates that the span describes consumer receiving a message from a broker. + // Like the PRODUCER kind, there is often no direct critical path latency relationship + // between producer and consumer spans. + SPAN_KIND_CONSUMER = 5; + } + + // Distinguishes between spans generated in a particular context. For example, + // two spans with the same name may be distinguished using `CLIENT` (caller) + // and `SERVER` (callee) to identify queueing latency associated with the span. + SpanKind kind = 6; + + // The start time of the span. On the client side, this is the time + // kept by the local machine where the span execution starts. On the server side, this + // is the time when the server's application handler starts running. + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // + // This field is semantically required and it is expected that end_time >= start_time. + fixed64 start_time_unix_nano = 7; + + // The end time of the span. On the client side, this is the time + // kept by the local machine where the span execution ends. On the server side, this + // is the time when the server application handler stops running. + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // + // This field is semantically required and it is expected that end_time >= start_time. + fixed64 end_time_unix_nano = 8; + + // A collection of key/value pairs. Note, global attributes + // like server name can be set using the resource API. Examples of attributes: + // + // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + // "/http/server_latency": 300 + // "example.com/myattribute": true + // "example.com/score": 10.239 + // + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; + + // The number of attributes that were discarded. Attributes + // can be discarded because their keys are too long or because there are too many + // attributes. If this value is 0, then no attributes were dropped. + uint32 dropped_attributes_count = 10; + + // Event is a time-stamped annotation of the span, consisting of user-supplied + // text description and key-value pairs. + message Event { + // The time the event occurred. + fixed64 time_unix_nano = 1; + + // The name of the event. + // This field is semantically required to be set to non-empty string. + string name = 2; + + // A collection of attribute key/value pairs on the event. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 3; + + // The number of dropped attributes. If the value is 0, + // then no attributes were dropped. + uint32 dropped_attributes_count = 4; + } + + // A collection of Event items. + repeated Event events = 11; + + // The number of dropped events. If the value is 0, then no + // events were dropped. + uint32 dropped_events_count = 12; + + // A pointer from the current span to another span in the same trace or in a + // different trace. For example, this can be used in batching operations, + // where a single batch handler processes multiple requests from different + // traces or when the handler receives a request from a different project. + message Link { + // A unique identifier of a trace that this linked span is part of. The ID is a + // 16-byte array. + bytes trace_id = 1; + + // A unique identifier for the linked span. The ID is an 8-byte array. + bytes span_id = 2; + + // The trace_state associated with the link. + string trace_state = 3; + + // A collection of attribute key/value pairs on the link. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 4; + + // The number of dropped attributes. If the value is 0, + // then no attributes were dropped. + uint32 dropped_attributes_count = 5; + + // Flags, a bit field. + // + // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + // Context specification. To read the 8-bit W3C trace flag, use + // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + // + // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. + // + // Bits 8 and 9 represent the 3 states of whether the link is remote. + // The states are (unknown, is not remote, is remote). + // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + // To read whether the link is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + // + // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + // When creating new spans, bits 10-31 (most-significant 22-bits) MUST be zero. + // + // [Optional]. + fixed32 flags = 6; + } + + // A collection of Links, which are references from this span to a span + // in the same or different trace. + repeated Link links = 13; + + // The number of dropped links after the maximum size was + // enforced. If this value is 0, then no links were dropped. + uint32 dropped_links_count = 14; + + // An optional final status for this span. Semantically when Status isn't set, it means + // span's status code is unset, i.e. assume STATUS_CODE_UNSET (code = 0). + Status status = 15; +} + +// The Status type defines a logical error model that is suitable for different +// programming environments, including REST APIs and RPC APIs. +message Status { + reserved 1; + + // A developer-facing human readable error message. + string message = 2; + + // For the semantics of status codes see + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#set-status + enum StatusCode { + // The default status. + STATUS_CODE_UNSET = 0; + // The Span has been validated by an Application developer or Operator to + // have completed successfully. + STATUS_CODE_OK = 1; + // The Span contains an error. + STATUS_CODE_ERROR = 2; + }; + + // The status code. + StatusCode code = 3; +} + +// SpanFlags represents constants used to interpret the +// Span.flags field, which is protobuf 'fixed32' type and is to +// be used as bit-fields. Each non-zero value defined in this enum is +// a bit-mask. To extract the bit-field, for example, use an +// expression like: +// +// (span.flags & SPAN_FLAGS_TRACE_FLAGS_MASK) +// +// See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. +// +// Note that Span flags were introduced in version 1.1 of the +// OpenTelemetry protocol. Older Span producers do not set this +// field, consequently consumers should not rely on the absence of a +// particular flag bit to indicate the presence of a particular feature. +enum SpanFlags { + // The zero value for the enum. Should not be used for comparisons. + // Instead use bitwise "and" with the appropriate mask as shown above. + SPAN_FLAGS_DO_NOT_USE = 0; + + // Bits 0-7 are used for trace flags. + SPAN_FLAGS_TRACE_FLAGS_MASK = 0x000000FF; + + // Bits 8 and 9 are used to indicate that the parent span or link span is remote. + // Bit 8 (`HAS_IS_REMOTE`) indicates whether the value is known. + // Bit 9 (`IS_REMOTE`) indicates whether the span or link is remote. + SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100; + SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 0x00000200; + + // Bits 10-31 are reserved for future use. +} \ No newline at end of file From 6f385bab2c1a1567e5c1ea3396b36c6c6f64af6c Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 14:14:16 -0400 Subject: [PATCH 04/17] feat(trace-utils): add serde->prost OTLP converter --- libdd-trace-utils/src/otlp_encoder/mod.rs | 1 + .../src/otlp_encoder/proto_convert.rs | 224 ++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 libdd-trace-utils/src/otlp_encoder/proto_convert.rs diff --git a/libdd-trace-utils/src/otlp_encoder/mod.rs b/libdd-trace-utils/src/otlp_encoder/mod.rs index 782a10e10d..733f870c72 100644 --- a/libdd-trace-utils/src/otlp_encoder/mod.rs +++ b/libdd-trace-utils/src/otlp_encoder/mod.rs @@ -5,6 +5,7 @@ pub mod json_types; pub mod mapper; +pub mod proto_convert; pub use mapper::map_traces_to_otlp; diff --git a/libdd-trace-utils/src/otlp_encoder/proto_convert.rs b/libdd-trace-utils/src/otlp_encoder/proto_convert.rs new file mode 100644 index 0000000000..d05654130d --- /dev/null +++ b/libdd-trace-utils/src/otlp_encoder/proto_convert.rs @@ -0,0 +1,224 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Converts the hand-rolled serde OTLP request (the JSON wire model) into the generated +//! prost types for binary (HTTP/protobuf) export. The semantic DD-span -> OTLP mapping already +//! happened in `mapper.rs`; this is a purely structural translation. + +use crate::otlp_encoder::json_types as j; +use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest as ProtoReq; +use libdd_trace_protobuf::opentelemetry::proto::common::v1::{ + any_value::Value as ProtoValue, AnyValue as ProtoAnyValue, ArrayValue as ProtoArrayValue, + InstrumentationScope as ProtoScope, KeyValue as ProtoKeyValue, +}; +use libdd_trace_protobuf::opentelemetry::proto::resource::v1::Resource as ProtoResource; +use libdd_trace_protobuf::opentelemetry::proto::trace::v1::{ + span::{Event as ProtoEvent, Link as ProtoLink}, + status::StatusCode as ProtoStatusCode, + ResourceSpans as ProtoResourceSpans, ScopeSpans as ProtoScopeSpans, Span as ProtoSpan, + Status as ProtoStatus, +}; + +/// Decode a fixed-width lowercase hex string into a byte vector. The mapper always produces +/// well-formed hex of the expected width; on a malformed value we fall back to an all-zero +/// buffer of `len` bytes rather than panicking (FFI reliability). +fn hex_to_bytes(s: &str, len: usize) -> Vec { + let bytes = s.as_bytes(); + if bytes.len() != len * 2 { + return vec![0u8; len]; + } + let mut out = Vec::with_capacity(len); + let mut i = 0; + while i < bytes.len() { + match (hex_nibble(bytes[i]), hex_nibble(bytes[i + 1])) { + (Some(hi), Some(lo)) => out.push((hi << 4) | lo), + _ => return vec![0u8; len], + } + i += 2; + } + out +} + +fn hex_nibble(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +fn parse_u64(s: &str) -> u64 { + s.parse().unwrap_or(0) +} + +impl From<&j::AnyValue> for ProtoAnyValue { + fn from(v: &j::AnyValue) -> Self { + let value = match v { + j::AnyValue::StringValue(s) => ProtoValue::StringValue(s.clone()), + j::AnyValue::BoolValue(b) => ProtoValue::BoolValue(*b), + j::AnyValue::IntValue(i) => ProtoValue::IntValue(*i), + j::AnyValue::DoubleValue(d) => ProtoValue::DoubleValue(*d), + j::AnyValue::BytesValue(b) => ProtoValue::BytesValue(b.clone()), + j::AnyValue::ArrayValue(a) => ProtoValue::ArrayValue(ProtoArrayValue { + values: a.values.iter().map(ProtoAnyValue::from).collect(), + }), + }; + ProtoAnyValue { value: Some(value) } + } +} + +fn kv(k: &j::KeyValue) -> ProtoKeyValue { + ProtoKeyValue { + key: k.key.clone(), + value: Some(ProtoAnyValue::from(&k.value)), + key_ref: 0, + } +} + +impl From<&j::ExportTraceServiceRequest> for ProtoReq { + fn from(req: &j::ExportTraceServiceRequest) -> Self { + ProtoReq { + resource_spans: req.resource_spans.iter().map(resource_spans).collect(), + } + } +} + +fn resource_spans(rs: &j::ResourceSpans) -> ProtoResourceSpans { + ProtoResourceSpans { + resource: rs.resource.as_ref().map(|r| ProtoResource { + attributes: r.attributes.iter().map(kv).collect(), + dropped_attributes_count: 0, + entity_refs: Vec::new(), + }), + scope_spans: rs.scope_spans.iter().map(scope_spans).collect(), + schema_url: String::new(), + } +} + +fn scope_spans(ss: &j::ScopeSpans) -> ProtoScopeSpans { + ProtoScopeSpans { + scope: ss.scope.as_ref().map(|s| ProtoScope { + name: s.name.clone().unwrap_or_default(), + version: s.version.clone().unwrap_or_default(), + attributes: Vec::new(), + dropped_attributes_count: 0, + }), + spans: ss.spans.iter().map(span).collect(), + schema_url: ss.schema_url.clone().unwrap_or_default(), + } +} + +fn span(s: &j::OtlpSpan) -> ProtoSpan { + ProtoSpan { + trace_id: hex_to_bytes(&s.trace_id, 16), + span_id: hex_to_bytes(&s.span_id, 8), + trace_state: s.trace_state.clone().unwrap_or_default(), + parent_span_id: s + .parent_span_id + .as_ref() + .map(|p| hex_to_bytes(p, 8)) + .unwrap_or_default(), + flags: s.flags.unwrap_or(0), + name: s.name.clone(), + kind: s.kind, + start_time_unix_nano: parse_u64(&s.start_time_unix_nano), + end_time_unix_nano: parse_u64(&s.end_time_unix_nano), + attributes: s.attributes.iter().map(kv).collect(), + dropped_attributes_count: s.dropped_attributes_count.unwrap_or(0), + events: s.events.iter().map(event).collect(), + dropped_events_count: s.dropped_events_count.unwrap_or(0), + links: s.links.iter().map(link).collect(), + dropped_links_count: 0, + status: Some(ProtoStatus { + message: s.status.message.clone().unwrap_or_default(), + code: status_code(s.status.code), + }), + } +} + +fn status_code(code: i32) -> i32 { + match code { + c if c == j::status_code::OK => ProtoStatusCode::Ok as i32, + c if c == j::status_code::ERROR => ProtoStatusCode::Error as i32, + _ => ProtoStatusCode::Unset as i32, + } +} + +fn link(l: &j::OtlpSpanLink) -> ProtoLink { + ProtoLink { + trace_id: hex_to_bytes(&l.trace_id, 16), + span_id: hex_to_bytes(&l.span_id, 8), + trace_state: l.trace_state.clone().unwrap_or_default(), + attributes: l.attributes.iter().map(kv).collect(), + dropped_attributes_count: l.dropped_attributes_count.unwrap_or(0), + flags: 0, + } +} + +fn event(e: &j::OtlpSpanEvent) -> ProtoEvent { + ProtoEvent { + time_unix_nano: parse_u64(&e.time_unix_nano), + name: e.name.clone(), + attributes: e.attributes.iter().map(kv).collect(), + dropped_attributes_count: e.dropped_attributes_count.unwrap_or(0), + } +} + +#[cfg(test)] +mod tests { + use crate::otlp_encoder::{map_traces_to_otlp, OtlpResourceInfo}; + use crate::span::BytesData; + use crate::span::v04::Span; + use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest as ProtoReq; + + #[test] + fn converts_ids_and_attributes_to_proto() { + let resource_info = OtlpResourceInfo { + service: "svc".to_string(), + ..Default::default() + }; + let mut span: Span = Span { + trace_id: 0xD269B633813FC60C_u128, + span_id: 0xEEE19B7EC3C1B174, + parent_id: 0xEEE19B7EC3C1B173, + name: libdd_tinybytes::BytesString::from_static("op"), + resource: libdd_tinybytes::BytesString::from_static("res"), + r#type: libdd_tinybytes::BytesString::from_static("web"), + start: 1544712660000000000, + duration: 1000000000, + error: 0, + ..Default::default() + }; + span.metrics + .insert(libdd_tinybytes::BytesString::from_static("count"), 42.0); + + let serde_req = map_traces_to_otlp(vec![vec![span]], &resource_info); + let proto: ProtoReq = (&serde_req).into(); + + let rs = &proto.resource_spans[0]; + let sp = &rs.scope_spans[0].spans[0]; + assert_eq!( + sp.trace_id, + vec![0, 0, 0, 0, 0, 0, 0, 0, 0xD2, 0x69, 0xB6, 0x33, 0x81, 0x3F, 0xC6, 0x0C] + ); + assert_eq!(sp.span_id, vec![0xEE, 0xE1, 0x9B, 0x7E, 0xC3, 0xC1, 0xB1, 0x74]); + assert_eq!( + sp.parent_span_id, + vec![0xEE, 0xE1, 0x9B, 0x7E, 0xC3, 0xC1, 0xB1, 0x73] + ); + assert_eq!(sp.name, "res"); + assert_eq!(sp.start_time_unix_nano, 1544712660000000000); + assert_eq!(sp.end_time_unix_nano, 1544712661000000000); + let count = sp + .attributes + .iter() + .find(|kv| kv.key == "count") + .expect("count attr"); + use libdd_trace_protobuf::opentelemetry::proto::common::v1::any_value::Value; + assert!(matches!( + count.value.as_ref().unwrap().value, + Some(Value::IntValue(42)) + )); + } +} From a52e30b73df58fba23684b2cc891fa2766380a2d Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 14:23:50 -0400 Subject: [PATCH 05/17] refactor(trace-utils): clarify OTLP converter + add fallback/status tests Co-Authored-By: Claude Sonnet 4.6 --- .../src/otlp_encoder/proto_convert.rs | 99 ++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/libdd-trace-utils/src/otlp_encoder/proto_convert.rs b/libdd-trace-utils/src/otlp_encoder/proto_convert.rs index d05654130d..a3658981d1 100644 --- a/libdd-trace-utils/src/otlp_encoder/proto_convert.rs +++ b/libdd-trace-utils/src/otlp_encoder/proto_convert.rs @@ -72,6 +72,10 @@ fn kv(k: &j::KeyValue) -> ProtoKeyValue { ProtoKeyValue { key: k.key.clone(), value: Some(ProtoAnyValue::from(&k.value)), + // `key_ref` and `entity_refs` (on Resource) are profiling-signal-only proto fields, + // unused for traces. Set explicitly to their zero defaults so the converter fails to + // compile if the proto shape changes (rather than silently misusing + // `..Default::default()`). key_ref: 0, } } @@ -89,6 +93,8 @@ fn resource_spans(rs: &j::ResourceSpans) -> ProtoResourceSpans { resource: rs.resource.as_ref().map(|r| ProtoResource { attributes: r.attributes.iter().map(kv).collect(), dropped_attributes_count: 0, + // `entity_refs` is a profiling-signal-only proto field, unused for traces. + // Explicit default (see `key_ref` note in `kv()`). entity_refs: Vec::new(), }), scope_spans: rs.scope_spans.iter().map(scope_spans).collect(), @@ -129,6 +135,8 @@ fn span(s: &j::OtlpSpan) -> ProtoSpan { events: s.events.iter().map(event).collect(), dropped_events_count: s.dropped_events_count.unwrap_or(0), links: s.links.iter().map(link).collect(), + // The serde `OtlpSpan` model does not track dropped links (the mapper enforces no + // link cap), so 0 is always correct here. dropped_links_count: 0, status: Some(ProtoStatus { message: s.status.message.clone().unwrap_or_default(), @@ -137,6 +145,13 @@ fn span(s: &j::OtlpSpan) -> ProtoSpan { } } +/// Map a serde status-code integer to its prost counterpart. +/// +/// The serde (`json_types::status_code`) and prost (`ProtoStatusCode`) numeric values are +/// intentionally identical — UNSET=0, OK=1, ERROR=2 — so each arm is a no-op in practice. +/// The explicit match is kept as a correctness guard: the `_` arm deliberately clamps any +/// unrecognized value (e.g. a future proto extension not yet reflected in the serde model) +/// to `Unset` rather than forwarding an out-of-range integer to the wire. fn status_code(code: i32) -> i32 { match code { c if c == j::status_code::OK => ProtoStatusCode::Ok as i32, @@ -152,6 +167,7 @@ fn link(l: &j::OtlpSpanLink) -> ProtoLink { trace_state: l.trace_state.clone().unwrap_or_default(), attributes: l.attributes.iter().map(kv).collect(), dropped_attributes_count: l.dropped_attributes_count.unwrap_or(0), + // `json_types::OtlpSpanLink` has no `flags` field, so 0 is the faithful value. flags: 0, } } @@ -167,10 +183,12 @@ fn event(e: &j::OtlpSpanEvent) -> ProtoEvent { #[cfg(test)] mod tests { + use super::hex_to_bytes; use crate::otlp_encoder::{map_traces_to_otlp, OtlpResourceInfo}; - use crate::span::BytesData; use crate::span::v04::Span; + use crate::span::BytesData; use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest as ProtoReq; + use libdd_trace_protobuf::opentelemetry::proto::trace::v1::status::StatusCode as ProtoStatusCode; #[test] fn converts_ids_and_attributes_to_proto() { @@ -202,7 +220,10 @@ mod tests { sp.trace_id, vec![0, 0, 0, 0, 0, 0, 0, 0, 0xD2, 0x69, 0xB6, 0x33, 0x81, 0x3F, 0xC6, 0x0C] ); - assert_eq!(sp.span_id, vec![0xEE, 0xE1, 0x9B, 0x7E, 0xC3, 0xC1, 0xB1, 0x74]); + assert_eq!( + sp.span_id, + vec![0xEE, 0xE1, 0x9B, 0x7E, 0xC3, 0xC1, 0xB1, 0x74] + ); assert_eq!( sp.parent_span_id, vec![0xEE, 0xE1, 0x9B, 0x7E, 0xC3, 0xC1, 0xB1, 0x73] @@ -221,4 +242,78 @@ mod tests { Some(Value::IntValue(42)) )); } + + // --- hex_to_bytes fallback tests --- + + #[test] + fn hex_to_bytes_wrong_length_returns_zeros() { + // "abc" is 3 chars but we expect 2 bytes (4 chars); should fall back to all-zero. + assert_eq!(hex_to_bytes("abc", 2), vec![0u8; 2]); + } + + #[test] + fn hex_to_bytes_bad_nibble_returns_zeros() { + // "zz" is the right length for 1 byte but contains invalid hex chars. + assert_eq!(hex_to_bytes("zz", 1), vec![0u8; 1]); + } + + // --- Status code + double metric test --- + + #[test] + fn error_span_produces_error_status_and_double_metric() { + // mapper.rs sets status.code = status_code::ERROR when span.error != 0, so + // proto_convert's status_code() must return ProtoStatusCode::Error as i32. + let resource_info = OtlpResourceInfo { + service: "svc-error-test".to_string(), + ..Default::default() + }; + let mut span: Span = Span { + trace_id: 0x1_u128, + span_id: 0x2, + name: libdd_tinybytes::BytesString::from_static("op"), + resource: libdd_tinybytes::BytesString::from_static("res"), + r#type: libdd_tinybytes::BytesString::from_static("web"), + start: 1_000_000_000, + duration: 500_000, + error: 1, // triggers ERROR status in mapper + ..Default::default() + }; + span.metrics + .insert(libdd_tinybytes::BytesString::from_static("ratio"), 1.5_f64); + + let serde_req = map_traces_to_otlp(vec![vec![span]], &resource_info); + let proto: ProtoReq = (&serde_req).into(); + + let sp = &proto.resource_spans[0].scope_spans[0].spans[0]; + + // (a) status code must be ERROR + assert_eq!( + sp.status.as_ref().unwrap().code, + ProtoStatusCode::Error as i32 + ); + + // (b) the "ratio" metric must arrive as a DoubleValue + use libdd_trace_protobuf::opentelemetry::proto::common::v1::any_value::Value; + let ratio_attr = sp + .attributes + .iter() + .find(|kv| kv.key == "ratio") + .expect("ratio attr must be present"); + assert!( + matches!( + ratio_attr.value.as_ref().unwrap().value, + Some(Value::DoubleValue(v)) if (v - 1.5).abs() < f64::EPSILON + ), + "expected DoubleValue(1.5), got {:?}", + ratio_attr.value + ); + } + + // Link/Event byte-size test: + // A plain v04 Span produced by the mapper does not carry links or events unless + // span.span_links / span.span_events are populated explicitly. Building a span with + // a link requires constructing a SpanLink with real trace_id/span_id values, which + // is straightforward, but the mapper only forwards links as-is — there is no + // transformation that would exercise proto_convert beyond what the ID tests above + // already cover. We therefore skip this sub-item as instructed. } From 4a4846af2133f451bc8668db37a49bd39fa385c0 Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 14:27:33 -0400 Subject: [PATCH 06/17] feat(trace-utils): add encode_otlp_json/encode_otlp_protobuf --- libdd-trace-utils/Cargo.toml | 1 + libdd-trace-utils/src/otlp_encoder/mod.rs | 66 +++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/libdd-trace-utils/Cargo.toml b/libdd-trace-utils/Cargo.toml index 1d9fa07fe7..01b8a6f744 100644 --- a/libdd-trace-utils/Cargo.toml +++ b/libdd-trace-utils/Cargo.toml @@ -70,6 +70,7 @@ libdd-common = { path = "../libdd-common", default-features = false, features = bolero = "0.13" criterion = "0.5.1" httpmock = { version = "0.8.0-alpha.1" } +hex = "0.4" serde_json = "1.0" tokio = { version = "1", features = ["macros", "rt-multi-thread", "test-util"] } libdd-trace-utils = { path = ".", features = ["test-utils"] } diff --git a/libdd-trace-utils/src/otlp_encoder/mod.rs b/libdd-trace-utils/src/otlp_encoder/mod.rs index 733f870c72..6c72f03493 100644 --- a/libdd-trace-utils/src/otlp_encoder/mod.rs +++ b/libdd-trace-utils/src/otlp_encoder/mod.rs @@ -7,8 +7,74 @@ pub mod json_types; pub mod mapper; pub mod proto_convert; +pub use json_types::ExportTraceServiceRequest; pub use mapper::map_traces_to_otlp; +use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest as ProtoExportTraceServiceRequest; +use prost::Message; + +/// Serialize an OTLP request to the HTTP/JSON wire format. +pub fn encode_otlp_json(req: &ExportTraceServiceRequest) -> serde_json::Result> { + serde_json::to_vec(req) +} + +/// Serialize an OTLP request to the HTTP/protobuf wire format. +pub fn encode_otlp_protobuf(req: &ExportTraceServiceRequest) -> Vec { + let proto: ProtoExportTraceServiceRequest = req.into(); + proto.encode_to_vec() +} + +#[cfg(test)] +mod encode_tests { + use super::*; + use crate::span::v04::Span; + use crate::span::BytesData; + use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest as ProtoReq; + use prost::Message; + + fn sample() -> ExportTraceServiceRequest { + let resource_info = OtlpResourceInfo { + service: "svc".to_string(), + ..Default::default() + }; + let span: Span = Span { + trace_id: 0xD269B633813FC60C_u128, + span_id: 0xEEE19B7EC3C1B174, + name: libdd_tinybytes::BytesString::from_static("op"), + resource: libdd_tinybytes::BytesString::from_static("res"), + start: 1, + duration: 2, + ..Default::default() + }; + map_traces_to_otlp(vec![vec![span]], &resource_info) + } + + #[test] + fn json_and_protobuf_carry_same_span() { + let req = sample(); + let json = encode_otlp_json(&req).unwrap(); + let pb = encode_otlp_protobuf(&req); + + let json_v: serde_json::Value = serde_json::from_slice(&json).unwrap(); + let json_name = json_v["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["name"] + .as_str() + .unwrap() + .to_string(); + + let proto = ProtoReq::decode(pb.as_slice()).unwrap(); + let proto_name = proto.resource_spans[0].scope_spans[0].spans[0].name.clone(); + + assert_eq!(json_name, "res"); + assert_eq!(proto_name, "res"); + let json_sid = json_v["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["spanId"] + .as_str() + .unwrap() + .to_string(); + let proto_sid = &proto.resource_spans[0].scope_spans[0].spans[0].span_id; + assert_eq!(json_sid, hex::encode(proto_sid)); + } +} + /// Tracer-level attributes used to populate the OTLP Resource on export. /// /// These are the fields from the tracer's configuration that map to OTLP Resource attributes From 54d885665d42b2335727adf2bda7ae472fb4f35d Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 14:45:23 -0400 Subject: [PATCH 07/17] feat(data-pipeline): make OtlpProtocol public with FromStr Co-Authored-By: Claude Opus 4.8 --- libdd-data-pipeline/src/lib.rs | 2 +- libdd-data-pipeline/src/otlp/config.rs | 43 +++++++++++++++++++++----- libdd-data-pipeline/src/otlp/mod.rs | 2 +- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/libdd-data-pipeline/src/lib.rs b/libdd-data-pipeline/src/lib.rs index 2b9955ce3d..f34613bd51 100644 --- a/libdd-data-pipeline/src/lib.rs +++ b/libdd-data-pipeline/src/lib.rs @@ -13,7 +13,7 @@ pub mod agent_info; mod health_metrics; -pub(crate) mod otlp; +pub mod otlp; #[cfg(feature = "telemetry")] pub(crate) mod telemetry; #[cfg(not(target_arch = "wasm32"))] diff --git a/libdd-data-pipeline/src/otlp/config.rs b/libdd-data-pipeline/src/otlp/config.rs index 02d7a45f80..42bf639f12 100644 --- a/libdd-data-pipeline/src/otlp/config.rs +++ b/libdd-data-pipeline/src/otlp/config.rs @@ -6,20 +6,31 @@ use http::HeaderMap; use std::time::Duration; -/// OTLP trace export protocol. HTTP/JSON is currently supported. +/// OTLP trace export protocol. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub(crate) enum OtlpProtocol { +pub enum OtlpProtocol { /// HTTP with JSON body (Content-Type: application/json). Default for HTTP. #[default] HttpJson, - /// HTTP with protobuf body. (Not supported yet) - #[allow(dead_code)] + /// HTTP with protobuf body (Content-Type: application/x-protobuf). HttpProtobuf, /// gRPC. (Not supported yet) #[allow(dead_code)] Grpc, } +impl std::str::FromStr for OtlpProtocol { + type Err = String; + fn from_str(s: &str) -> Result { + match s { + "http/json" => Ok(OtlpProtocol::HttpJson), + "http/protobuf" => Ok(OtlpProtocol::HttpProtobuf), + "grpc" => Ok(OtlpProtocol::Grpc), + other => Err(format!("unknown OTLP protocol: {other}")), + } + } +} + /// Default timeout for OTLP export requests. pub const DEFAULT_OTLP_TIMEOUT: Duration = Duration::from_secs(10); @@ -32,7 +43,25 @@ pub struct OtlpTraceConfig { pub headers: HeaderMap, /// Request timeout. pub timeout: Duration, - /// Protocol (for future use; currently only HttpJson is supported). - #[allow(dead_code)] - pub(crate) protocol: OtlpProtocol, + /// OTLP export protocol (selects body encoding and content-type). + pub protocol: OtlpProtocol, +} + +#[cfg(test)] +mod tests { + use super::*; + use std::str::FromStr; + #[test] + fn protocol_from_str() { + assert_eq!( + OtlpProtocol::from_str("http/json").unwrap(), + OtlpProtocol::HttpJson + ); + assert_eq!( + OtlpProtocol::from_str("http/protobuf").unwrap(), + OtlpProtocol::HttpProtobuf + ); + assert_eq!(OtlpProtocol::from_str("grpc").unwrap(), OtlpProtocol::Grpc); + assert!(OtlpProtocol::from_str("nonsense").is_err()); + } } diff --git a/libdd-data-pipeline/src/otlp/mod.rs b/libdd-data-pipeline/src/otlp/mod.rs index 658fc13b87..690a6f5ff4 100644 --- a/libdd-data-pipeline/src/otlp/mod.rs +++ b/libdd-data-pipeline/src/otlp/mod.rs @@ -25,6 +25,6 @@ pub mod config; pub mod exporter; -pub use config::OtlpTraceConfig; +pub use config::{OtlpProtocol, OtlpTraceConfig}; pub use exporter::send_otlp_traces_http; pub use libdd_trace_utils::otlp_encoder::{map_traces_to_otlp, OtlpResourceInfo}; From 772be3e058c5a7a47c954bf63fd427d6280232ab Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 14:45:23 -0400 Subject: [PATCH 08/17] feat(data-pipeline): set OTLP content-type from protocol Co-Authored-By: Claude Opus 4.8 --- libdd-data-pipeline/src/otlp/exporter.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/libdd-data-pipeline/src/otlp/exporter.rs b/libdd-data-pipeline/src/otlp/exporter.rs index 1f4d86a235..e8498cedc3 100644 --- a/libdd-data-pipeline/src/otlp/exporter.rs +++ b/libdd-data-pipeline/src/otlp/exporter.rs @@ -16,7 +16,9 @@ const OTLP_MAX_RETRIES: u32 = 4; /// Initial backoff between retries (milliseconds). const OTLP_RETRY_DELAY_MS: u64 = 100; -/// Send OTLP trace payload (JSON bytes) to the configured endpoint with retries. +/// Send an OTLP trace payload to the configured endpoint with retries. +/// +/// The body encoding and `Content-Type` are selected from `config.protocol`. /// /// Uses [`send_with_retry`] for consistent retry behaviour and observability across exporters. /// @@ -26,7 +28,7 @@ pub async fn send_otlp_traces_http( capabilities: &C, config: &OtlpTraceConfig, test_token: Option<&str>, - json_body: Vec, + body: Vec, ) -> Result<(), TraceExporterError> { let url = libdd_common::parse_uri(&config.endpoint_url).map_err(|e| { TraceExporterError::Internal(InternalErrorKind::InvalidWorkerState(format!( @@ -41,11 +43,15 @@ pub async fn send_otlp_traces_http( ..Endpoint::default() }; + let content_type = match config.protocol { + crate::otlp::config::OtlpProtocol::HttpProtobuf => { + libdd_common::header::APPLICATION_PROTOBUF + } + _ => libdd_common::header::APPLICATION_JSON, + }; + let mut headers = config.headers.clone(); - headers.insert( - http::header::CONTENT_TYPE, - libdd_common::header::APPLICATION_JSON, - ); + headers.insert(http::header::CONTENT_TYPE, content_type); if let Some(token) = test_token { if let Ok(val) = http::HeaderValue::from_str(token) { headers.insert( @@ -62,7 +68,7 @@ pub async fn send_otlp_traces_http( None, ); - match send_with_retry(capabilities, &target, json_body, &headers, &retry_strategy).await { + match send_with_retry(capabilities, &target, body, &headers, &retry_strategy).await { Ok(_) => Ok(()), Err(e) => Err(map_send_error(e).await), } From 46d72a70adfb9fc3fc32c8a49ecbb7ba5c688bed Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 14:45:23 -0400 Subject: [PATCH 09/17] feat(data-pipeline): add TraceExporterBuilder::set_otlp_protocol Co-Authored-By: Claude Opus 4.8 --- libdd-data-pipeline/src/trace_exporter/builder.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/libdd-data-pipeline/src/trace_exporter/builder.rs b/libdd-data-pipeline/src/trace_exporter/builder.rs index 3c0e1f14b5..6f22c880cb 100644 --- a/libdd-data-pipeline/src/trace_exporter/builder.rs +++ b/libdd-data-pipeline/src/trace_exporter/builder.rs @@ -65,6 +65,7 @@ pub struct TraceExporterBuilder { connection_timeout: Option, otlp_endpoint: Option, otlp_headers: Vec<(String, String)>, + otlp_protocol: OtlpProtocol, } impl TraceExporterBuilder { @@ -286,6 +287,14 @@ impl TraceExporterBuilder { self } + /// Selects the OTLP export protocol. Accepts `OtlpProtocol::HttpJson` (default) or + /// `OtlpProtocol::HttpProtobuf`. The host language resolves this from + /// `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` / `OTEL_EXPORTER_OTLP_PROTOCOL`. + pub fn set_otlp_protocol(&mut self, protocol: OtlpProtocol) -> &mut Self { + self.otlp_protocol = protocol; + self + } + /// Sets additional HTTP headers to include in OTLP trace export requests. /// /// Headers should be provided as key-value pairs. The host language is responsible for @@ -451,7 +460,7 @@ impl TraceExporterBuilder { .connection_timeout .map(Duration::from_millis) .unwrap_or(DEFAULT_OTLP_TIMEOUT), - protocol: OtlpProtocol::HttpJson, + protocol: self.otlp_protocol, } }); From 8f3c38e9f667c1843a2185eb7767fff8ad880477 Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 14:45:23 -0400 Subject: [PATCH 10/17] feat(data-pipeline): dispatch OTLP encoder by protocol + protobuf test Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 2 + libdd-data-pipeline/Cargo.toml | 1 + libdd-data-pipeline/src/trace_exporter/mod.rs | 27 ++++-- ...est_trace_exporter_otlp_protobuf_export.rs | 86 +++++++++++++++++++ 4 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 libdd-data-pipeline/tests/test_trace_exporter_otlp_protobuf_export.rs diff --git a/Cargo.lock b/Cargo.lock index fff06dbaa3..333063960c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3006,6 +3006,7 @@ dependencies = [ "libdd-trace-protobuf", "libdd-trace-stats", "libdd-trace-utils", + "prost", "rand 0.8.5", "regex", "rmp-serde", @@ -3455,6 +3456,7 @@ dependencies = [ "flate2", "futures", "getrandom 0.2.15", + "hex", "http", "http-body", "http-body-util", diff --git a/libdd-data-pipeline/Cargo.toml b/libdd-data-pipeline/Cargo.toml index bb93a10a59..85681e902f 100644 --- a/libdd-data-pipeline/Cargo.toml +++ b/libdd-data-pipeline/Cargo.toml @@ -73,6 +73,7 @@ libdd-trace-utils = { path = "../libdd-trace-utils", features = [ "test-utils", ] } httpmock = "0.8.0-alpha.1" +prost = "0.14.1" rand = "0.8.5" tempfile = "3.3.0" tokio = { version = "1.23", features = [ diff --git a/libdd-data-pipeline/src/trace_exporter/mod.rs b/libdd-data-pipeline/src/trace_exporter/mod.rs index 070fc754e0..10d5ac95f1 100644 --- a/libdd-data-pipeline/src/trace_exporter/mod.rs +++ b/libdd-data-pipeline/src/trace_exporter/mod.rs @@ -15,7 +15,9 @@ use self::metrics::MetricsEmitter; use self::stats::StatsComputationStatus; use self::trace_serializer::TraceSerializer; use crate::agent_info::ResponseObserver; -use crate::otlp::{map_traces_to_otlp, send_otlp_traces_http, OtlpResourceInfo, OtlpTraceConfig}; +use crate::otlp::{ + map_traces_to_otlp, send_otlp_traces_http, OtlpProtocol, OtlpResourceInfo, OtlpTraceConfig, +}; #[cfg(feature = "telemetry")] use crate::telemetry::{SendPayloadTelemetry, TelemetryClient}; use crate::trace_exporter::agent_response::{ @@ -546,15 +548,28 @@ impl Tra r }; let request = map_traces_to_otlp(traces, &resource_info); - let json_body = serde_json::to_vec(&request).map_err(|e| { - error!("OTLP JSON serialization error: {e}"); - TraceExporterError::Internal(InternalErrorKind::InvalidWorkerState(e.to_string())) - })?; + let body = match config.protocol { + OtlpProtocol::HttpJson => libdd_trace_utils::otlp_encoder::encode_otlp_json(&request) + .map_err(|e| { + error!("OTLP JSON serialization error: {e}"); + TraceExporterError::Internal(InternalErrorKind::InvalidWorkerState(e.to_string())) + })?, + OtlpProtocol::HttpProtobuf => { + libdd_trace_utils::otlp_encoder::encode_otlp_protobuf(&request) + } + OtlpProtocol::Grpc => { + return Err(TraceExporterError::Internal( + InternalErrorKind::InvalidWorkerState( + "OTLP gRPC export is not supported".to_string(), + ), + )); + } + }; send_otlp_traces_http( &self.capabilities, config, self.endpoint.test_token.as_deref(), - json_body, + body, ) .await?; Ok(AgentResponse::Unchanged) diff --git a/libdd-data-pipeline/tests/test_trace_exporter_otlp_protobuf_export.rs b/libdd-data-pipeline/tests/test_trace_exporter_otlp_protobuf_export.rs new file mode 100644 index 0000000000..4bff160dd4 --- /dev/null +++ b/libdd-data-pipeline/tests/test_trace_exporter_otlp_protobuf_export.rs @@ -0,0 +1,86 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 +#[cfg(test)] +mod otlp_protobuf_tests { + use libdd_capabilities_impl::NativeCapabilities; + use libdd_data_pipeline::trace_exporter::TraceExporterBuilder; + use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest; + use libdd_trace_utils::test_utils::create_test_json_span; + use prost::Message; + use serde_json::json; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + use tokio::task; + + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn otlp_protobuf_export_sends_decodable_payload() { + use httpmock::MockServer; + + // The httpmock 0.8 alpha API does not expose captured request bodies after the fact, so + // we decode and validate the protobuf body inside a custom request matcher. The matcher + // flips `body_valid` when the payload decodes and carries the expected service.name. + let body_valid = Arc::new(AtomicBool::new(false)); + let matcher_flag = body_valid.clone(); + + let server = MockServer::start_async().await; + let mock = server + .mock_async(move |when, then| { + let flag = matcher_flag.clone(); + when.method("POST") + .path("/v1/traces") + .header("content-type", "application/x-protobuf") + .is_true(move |req: &httpmock::prelude::HttpMockRequest| { + use libdd_trace_protobuf::opentelemetry::proto::common::v1::any_value::Value; + let Ok(decoded) = ExportTraceServiceRequest::decode(req.body_ref()) else { + return false; + }; + let valid = decoded + .resource_spans + .first() + .and_then(|rs| rs.resource.as_ref()) + .map(|resource| { + resource.attributes.iter().any(|kv| { + kv.key == "service.name" + && matches!( + kv.value.as_ref().and_then(|v| v.value.as_ref()), + Some(Value::StringValue(s)) if s == "test" + ) + }) + }) + .unwrap_or(false); + if valid { + flag.store(true, Ordering::SeqCst); + } + valid + }); + then.status(200).body(""); + }) + .await; + + let endpoint = format!("http://localhost:{}/v1/traces", server.port()); + let task_result = task::spawn_blocking(move || { + let mut builder = TraceExporterBuilder::default(); + builder + .set_otlp_endpoint(&endpoint) + .set_otlp_protocol(libdd_data_pipeline::otlp::config::OtlpProtocol::HttpProtobuf) + .set_language("test-lang") + .set_tracer_version("1.0") + .set_env("test_env") + .set_service("test"); + let exporter = builder.build::().expect("build"); + let mut span = create_test_json_span(1234, 12342, 12341, 1, false); + span["name"] = json!("pb_span"); + let data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); + exporter.send(data.as_ref()).expect("send ok"); + }) + .await; + + assert!(task_result.is_ok()); + assert_eq!(mock.calls_async().await, 1); + assert!( + body_valid.load(Ordering::SeqCst), + "protobuf body did not decode to the expected ExportTraceServiceRequest" + ); + } +} From 263342780124c170d88338e6fd5e957dcd90a1f0 Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 14:57:12 -0400 Subject: [PATCH 11/17] refactor(data-pipeline): narrow otlp pub surface + exhaustive content-type match Co-Authored-By: Claude Sonnet 4.6 --- libdd-data-pipeline/src/otlp/config.rs | 4 ++-- libdd-data-pipeline/src/otlp/exporter.rs | 4 +++- libdd-data-pipeline/src/otlp/mod.rs | 11 ++++++----- .../tests/test_trace_exporter_otlp_protobuf_export.rs | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/libdd-data-pipeline/src/otlp/config.rs b/libdd-data-pipeline/src/otlp/config.rs index 42bf639f12..e48f3961bd 100644 --- a/libdd-data-pipeline/src/otlp/config.rs +++ b/libdd-data-pipeline/src/otlp/config.rs @@ -14,8 +14,8 @@ pub enum OtlpProtocol { HttpJson, /// HTTP with protobuf body (Content-Type: application/x-protobuf). HttpProtobuf, - /// gRPC. (Not supported yet) - #[allow(dead_code)] + /// gRPC. Parsed by `FromStr` so callers get a clean error, but rejected at export time + /// (unsupported). Grpc, } diff --git a/libdd-data-pipeline/src/otlp/exporter.rs b/libdd-data-pipeline/src/otlp/exporter.rs index e8498cedc3..929e42fa10 100644 --- a/libdd-data-pipeline/src/otlp/exporter.rs +++ b/libdd-data-pipeline/src/otlp/exporter.rs @@ -47,7 +47,9 @@ pub async fn send_otlp_traces_http( crate::otlp::config::OtlpProtocol::HttpProtobuf => { libdd_common::header::APPLICATION_PROTOBUF } - _ => libdd_common::header::APPLICATION_JSON, + crate::otlp::config::OtlpProtocol::HttpJson | crate::otlp::config::OtlpProtocol::Grpc => { + libdd_common::header::APPLICATION_JSON + } }; let mut headers = config.headers.clone(); diff --git a/libdd-data-pipeline/src/otlp/mod.rs b/libdd-data-pipeline/src/otlp/mod.rs index 690a6f5ff4..adde33396b 100644 --- a/libdd-data-pipeline/src/otlp/mod.rs +++ b/libdd-data-pipeline/src/otlp/mod.rs @@ -5,8 +5,9 @@ //! //! When an OTLP endpoint is configured via //! [`crate::trace_exporter::TraceExporterBuilder::set_otlp_endpoint`], the trace exporter sends -//! traces in OTLP HTTP/JSON format to that endpoint instead of the Datadog agent. The host language -//! is responsible for resolving the endpoint from its own configuration (e.g. +//! traces in OTLP HTTP format to that endpoint instead of the Datadog agent; the wire encoding +//! (JSON or protobuf) is selected via [`OtlpProtocol`]. The host language is responsible for +//! resolving the endpoint from its own configuration (e.g. //! `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`). //! //! ## Sampling @@ -22,9 +23,9 @@ //! spans from a local trace are closed (i.e. send complete trace chunks). This crate does not //! buffer or flush partially—it exports whatever trace chunks it receives. -pub mod config; -pub mod exporter; +pub(crate) mod config; +pub(crate) mod exporter; pub use config::{OtlpProtocol, OtlpTraceConfig}; -pub use exporter::send_otlp_traces_http; +pub(crate) use exporter::send_otlp_traces_http; pub use libdd_trace_utils::otlp_encoder::{map_traces_to_otlp, OtlpResourceInfo}; diff --git a/libdd-data-pipeline/tests/test_trace_exporter_otlp_protobuf_export.rs b/libdd-data-pipeline/tests/test_trace_exporter_otlp_protobuf_export.rs index 4bff160dd4..2d193ed387 100644 --- a/libdd-data-pipeline/tests/test_trace_exporter_otlp_protobuf_export.rs +++ b/libdd-data-pipeline/tests/test_trace_exporter_otlp_protobuf_export.rs @@ -63,7 +63,7 @@ mod otlp_protobuf_tests { let mut builder = TraceExporterBuilder::default(); builder .set_otlp_endpoint(&endpoint) - .set_otlp_protocol(libdd_data_pipeline::otlp::config::OtlpProtocol::HttpProtobuf) + .set_otlp_protocol(libdd_data_pipeline::otlp::OtlpProtocol::HttpProtobuf) .set_language("test-lang") .set_tracer_version("1.0") .set_env("test_env") From e493d8d9fad98d94fdac26143cb402e3518aa1e0 Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 15:00:09 -0400 Subject: [PATCH 12/17] feat(data-pipeline-ffi): add ddog_trace_exporter_config_set_otlp_protocol --- libdd-data-pipeline-ffi/src/trace_exporter.rs | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/libdd-data-pipeline-ffi/src/trace_exporter.rs b/libdd-data-pipeline-ffi/src/trace_exporter.rs index 5271c86e63..3490926c92 100644 --- a/libdd-data-pipeline-ffi/src/trace_exporter.rs +++ b/libdd-data-pipeline-ffi/src/trace_exporter.rs @@ -83,6 +83,7 @@ pub struct TraceExporterConfig { connection_timeout: Option, shared_runtime: Option>, otlp_endpoint: Option, + otlp_protocol: Option, } #[no_mangle] @@ -498,6 +499,34 @@ pub unsafe extern "C" fn ddog_trace_exporter_config_set_otlp_endpoint( ) } +/// Sets the OTLP export protocol. Accepts the OTel-standard values `http/json` (default) or +/// `http/protobuf`. `grpc` is rejected as not yet supported. The host language is responsible for +/// resolving the value (e.g. `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`). +#[no_mangle] +pub unsafe extern "C" fn ddog_trace_exporter_config_set_otlp_protocol( + config: Option<&mut TraceExporterConfig>, + protocol: CharSlice, +) -> Option> { + catch_panic!( + if let Some(handle) = config { + let value = match sanitize_string(protocol) { + Ok(s) => s, + Err(e) => return Some(e), + }; + match value.as_str() { + "http/json" | "http/protobuf" => { + handle.otlp_protocol = Some(value); + None + } + _ => gen_error!(ErrorCode::InvalidArgument), + } + } else { + gen_error!(ErrorCode::InvalidArgument) + }, + gen_error!(ErrorCode::Panic) + ) +} + /// Create a new TraceExporter instance. /// /// When an OTLP endpoint is configured via `TraceExporterConfig`, the exporter sends traces in @@ -565,6 +594,11 @@ pub unsafe extern "C" fn ddog_trace_exporter_new( if let Some(ref url) = config.otlp_endpoint { builder.set_otlp_endpoint(url); + if let Some(ref proto) = config.otlp_protocol { + if let Ok(p) = proto.parse::() { + builder.set_otlp_protocol(p); + } + } } match builder.build() { From 4a81412d9bc1a80e87068c791c8979c1daa97edb Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 15:04:41 -0400 Subject: [PATCH 13/17] test(data-pipeline-ffi): cover set_otlp_protocol + clarify contract Co-Authored-By: Claude Sonnet 4.6 --- libdd-data-pipeline-ffi/src/trace_exporter.rs | 72 ++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/libdd-data-pipeline-ffi/src/trace_exporter.rs b/libdd-data-pipeline-ffi/src/trace_exporter.rs index 3490926c92..409ad0e2f2 100644 --- a/libdd-data-pipeline-ffi/src/trace_exporter.rs +++ b/libdd-data-pipeline-ffi/src/trace_exporter.rs @@ -500,8 +500,11 @@ pub unsafe extern "C" fn ddog_trace_exporter_config_set_otlp_endpoint( } /// Sets the OTLP export protocol. Accepts the OTel-standard values `http/json` (default) or -/// `http/protobuf`. `grpc` is rejected as not yet supported. The host language is responsible for -/// resolving the value (e.g. `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`). +/// `http/protobuf`; `grpc` is rejected as not yet supported. The host language resolves the value +/// (e.g. from `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`). +/// +/// Returns `None` on success, `ErrorCode::InvalidArgument` for a null config or an unaccepted +/// value, and `ErrorCode::InvalidInput` for a non-UTF-8 string. #[no_mangle] pub unsafe extern "C" fn ddog_trace_exporter_config_set_otlp_protocol( config: Option<&mut TraceExporterConfig>, @@ -595,6 +598,8 @@ pub unsafe extern "C" fn ddog_trace_exporter_new( if let Some(ref url) = config.otlp_endpoint { builder.set_otlp_endpoint(url); if let Some(ref proto) = config.otlp_protocol { + // The FFI setter only stores "http/json"/"http/protobuf", so this parse always + // succeeds here; a parse failure just leaves the builder's default protocol. if let Ok(p) = proto.parse::() { builder.set_otlp_protocol(p); } @@ -1317,6 +1322,69 @@ mod tests { } } + #[test] + fn config_otlp_protocol_test() { + unsafe { + // Null config → InvalidArgument + let error = + ddog_trace_exporter_config_set_otlp_protocol(None, CharSlice::from("http/json")); + assert_eq!(error.as_ref().unwrap().code, ErrorCode::InvalidArgument); + ddog_trace_exporter_error_free(error); + + // "http/json" → success, stored + let mut config = Some(TraceExporterConfig::default()); + let error = ddog_trace_exporter_config_set_otlp_protocol( + config.as_mut(), + CharSlice::from("http/json"), + ); + assert_eq!(error, None); + assert_eq!( + config.as_ref().unwrap().otlp_protocol.as_deref(), + Some("http/json") + ); + + // "http/protobuf" → success, stored + let mut config = Some(TraceExporterConfig::default()); + let error = ddog_trace_exporter_config_set_otlp_protocol( + config.as_mut(), + CharSlice::from("http/protobuf"), + ); + assert_eq!(error, None); + assert_eq!( + config.as_ref().unwrap().otlp_protocol.as_deref(), + Some("http/protobuf") + ); + + // "grpc" → InvalidArgument + let mut config = Some(TraceExporterConfig::default()); + let error = ddog_trace_exporter_config_set_otlp_protocol( + config.as_mut(), + CharSlice::from("grpc"), + ); + assert_eq!(error.as_ref().unwrap().code, ErrorCode::InvalidArgument); + ddog_trace_exporter_error_free(error); + + // Garbage value → InvalidArgument + let mut config = Some(TraceExporterConfig::default()); + let error = ddog_trace_exporter_config_set_otlp_protocol( + config.as_mut(), + CharSlice::from("nonsense"), + ); + assert_eq!(error.as_ref().unwrap().code, ErrorCode::InvalidArgument); + ddog_trace_exporter_error_free(error); + + // Non-UTF-8 input → InvalidInput + let mut config = Some(TraceExporterConfig::default()); + let invalid: [u8; 2] = [0x80u8, 0xFFu8]; + let error = ddog_trace_exporter_config_set_otlp_protocol( + config.as_mut(), + CharSlice::from_bytes(&invalid), + ); + assert_eq!(error.as_ref().unwrap().code, ErrorCode::InvalidInput); + ddog_trace_exporter_error_free(error); + } + } + #[cfg(all(feature = "catch_panic", panic = "unwind"))] #[test] fn catch_panic_test() { From 3091b575df5aa550c1675fcab3326da3d95c95ff Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 15:14:05 -0400 Subject: [PATCH 14/17] fix(trace-protobuf): disable comments on vendored OTLP trace protos to avoid doctest failures --- libdd-trace-protobuf/build.rs | 9 + .../opentelemetry.proto.collector.trace.v1.rs | 31 --- .../src/opentelemetry.proto.trace.v1.rs | 214 ------------------ 3 files changed, 9 insertions(+), 245 deletions(-) diff --git a/libdd-trace-protobuf/build.rs b/libdd-trace-protobuf/build.rs index ba99a075a9..aee2ebd246 100644 --- a/libdd-trace-protobuf/build.rs +++ b/libdd-trace-protobuf/build.rs @@ -36,6 +36,15 @@ fn generate_protobuf() { config.out_dir(output_path.clone()); + // The vendored OpenTelemetry trace protos carry doc comments with indented example blocks + // (e.g. on `Span.attributes`) that rustdoc would interpret as Rust doctests and fail to + // compile. Drop the generated comments for these packages; the vendored `.proto` files remain + // the documentation source of truth. + config.disable_comments([ + ".opentelemetry.proto.trace.v1", + ".opentelemetry.proto.collector.trace.v1", + ]); + // The following prost_build config changes modify the protobuf generated structs in // in the following ways: diff --git a/libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs b/libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs index 3a1e3db44d..fbd03366a9 100644 --- a/libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs +++ b/libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs @@ -4,11 +4,6 @@ // This file is @generated by prost-build. #[derive(Clone, PartialEq, ::prost::Message)] pub struct ExportTraceServiceRequest { - /// An array of ResourceSpans. - /// For data coming from a single resource this array will typically contain one - /// element. Intermediary nodes (such as OpenTelemetry Collector) that receive - /// data from multiple origins typically batch the data before forwarding further and - /// in that case this array will contain multiple elements. #[prost(message, repeated, tag = "1")] pub resource_spans: ::prost::alloc::vec::Vec< super::super::super::trace::v1::ResourceSpans, @@ -16,39 +11,13 @@ pub struct ExportTraceServiceRequest { } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ExportTraceServiceResponse { - /// The details of a partially successful export request. - /// - /// If the request is only partially accepted - /// (i.e. when the server accepts only parts of the data and rejects the rest) - /// the server MUST initialize the `partial_success` field and MUST - /// set the `rejected_` with the number of items it rejected. - /// - /// Servers MAY also make use of the `partial_success` field to convey - /// warnings/suggestions to senders even when the request was fully accepted. - /// In such cases, the `rejected_` MUST have a value of `0` and - /// the `error_message` MUST be non-empty. - /// - /// A `partial_success` message with an empty value (rejected_ = 0 and - /// `error_message` = "") is equivalent to it not being set/present. Senders - /// SHOULD interpret it the same way as in the full success case. #[prost(message, optional, tag = "1")] pub partial_success: ::core::option::Option, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ExportTracePartialSuccess { - /// The number of rejected spans. - /// - /// A `rejected_` field holding a `0` value indicates that the - /// request was fully accepted. #[prost(int64, tag = "1")] pub rejected_spans: i64, - /// A developer-facing human-readable message in English. It should be used - /// either to explain why the server rejected parts of the data during a partial - /// success or to convey warnings/suggestions during a full success. The message - /// should offer guidance on how users can address such issues. - /// - /// error_message is an optional field. An error_message with an empty value - /// is equivalent to it not being set. #[prost(string, tag = "2")] pub error_message: ::prost::alloc::string::String, } diff --git a/libdd-trace-protobuf/src/opentelemetry.proto.trace.v1.rs b/libdd-trace-protobuf/src/opentelemetry.proto.trace.v1.rs index 2e6b3ec3a7..d1d035f759 100644 --- a/libdd-trace-protobuf/src/opentelemetry.proto.trace.v1.rs +++ b/libdd-trace-protobuf/src/opentelemetry.proto.trace.v1.rs @@ -2,266 +2,96 @@ // SPDX-License-Identifier: Apache-2.0 // This file is @generated by prost-build. -/// TracesData represents the traces data that can be stored in a persistent storage, -/// OR can be embedded by other protocols that transfer OTLP traces data but do -/// not implement the OTLP protocol. -/// -/// The main difference between this message and collector protocol is that -/// in this message there will not be any "control" or "metadata" specific to -/// OTLP protocol. -/// -/// When new fields are added into this message, the OTLP request MUST be updated -/// as well. #[derive(Clone, PartialEq, ::prost::Message)] pub struct TracesData { - /// An array of ResourceSpans. - /// For data coming from a single resource this array will typically contain - /// one element. Intermediary nodes that receive data from multiple origins - /// typically batch the data before forwarding further and in that case this - /// array will contain multiple elements. #[prost(message, repeated, tag = "1")] pub resource_spans: ::prost::alloc::vec::Vec, } -/// A collection of ScopeSpans from a Resource. #[derive(Clone, PartialEq, ::prost::Message)] pub struct ResourceSpans { - /// The resource for the spans in this message. - /// If this field is not set then no resource info is known. #[prost(message, optional, tag = "1")] pub resource: ::core::option::Option, - /// A list of ScopeSpans that originate from a resource. #[prost(message, repeated, tag = "2")] pub scope_spans: ::prost::alloc::vec::Vec, - /// The Schema URL, if known. This is the identifier of the Schema that the resource data - /// is recorded in. Notably, the last part of the URL path is the version number of the - /// schema: http\[s\]://server\[:port\]/path/. To learn more about Schema URL see - /// - /// This schema_url applies to the data in the "resource" field. It does not apply - /// to the data in the "scope_spans" field which have their own schema_url field. #[prost(string, tag = "3")] pub schema_url: ::prost::alloc::string::String, } -/// A collection of Spans produced by an InstrumentationScope. #[derive(Clone, PartialEq, ::prost::Message)] pub struct ScopeSpans { - /// The instrumentation scope information for the spans in this message. - /// Semantically when InstrumentationScope isn't set, it is equivalent with - /// an empty instrumentation scope name (unknown). #[prost(message, optional, tag = "1")] pub scope: ::core::option::Option, - /// A list of Spans that originate from an instrumentation scope. #[prost(message, repeated, tag = "2")] pub spans: ::prost::alloc::vec::Vec, - /// The Schema URL, if known. This is the identifier of the Schema that the span data - /// is recorded in. Notably, the last part of the URL path is the version number of the - /// schema: http\[s\]://server\[:port\]/path/. To learn more about Schema URL see - /// - /// This schema_url applies to the data in the "scope" field and all spans and span - /// events in the "spans" field. #[prost(string, tag = "3")] pub schema_url: ::prost::alloc::string::String, } -/// A Span represents a single operation performed by a single component of the system. -/// -/// The next available field id is 17. #[derive(Clone, PartialEq, ::prost::Message)] pub struct Span { - /// A unique identifier for a trace. All spans from the same trace share - /// the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR - /// of length other than 16 bytes is considered invalid (empty string in OTLP/JSON - /// is zero-length and thus is also invalid). - /// - /// This field is required. #[prost(bytes = "vec", tag = "1")] pub trace_id: ::prost::alloc::vec::Vec, - /// A unique identifier for a span within a trace, assigned when the span - /// is created. The ID is an 8-byte array. An ID with all zeroes OR of length - /// other than 8 bytes is considered invalid (empty string in OTLP/JSON - /// is zero-length and thus is also invalid). - /// - /// This field is required. #[prost(bytes = "vec", tag = "2")] pub span_id: ::prost::alloc::vec::Vec, - /// trace_state conveys information about request position in multiple distributed tracing graphs. - /// It is a trace_state in w3c-trace-context format: - /// See also for more details about this field. #[prost(string, tag = "3")] pub trace_state: ::prost::alloc::string::String, - /// The `span_id` of this span's parent span. If this is a root span, then this - /// field must be empty. The ID is an 8-byte array. #[prost(bytes = "vec", tag = "4")] pub parent_span_id: ::prost::alloc::vec::Vec, - /// Flags, a bit field. - /// - /// Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace - /// Context specification. To read the 8-bit W3C trace flag, use - /// `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. - /// - /// See for the flag definitions. - /// - /// Bits 8 and 9 represent the 3 states of whether a span's parent - /// is remote. The states are (unknown, is not remote, is remote). - /// To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. - /// To read whether the span is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. - /// - /// When creating span messages, if the message is logically forwarded from another source - /// with an equivalent flags fields (i.e., usually another OTLP span message), the field SHOULD - /// be copied as-is. If creating from a source that does not have an equivalent flags field - /// (such as a runtime representation of an OpenTelemetry span), the high 22 bits MUST - /// be set to zero. - /// Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. - /// - /// \[Optional\]. #[prost(fixed32, tag = "16")] pub flags: u32, - /// A description of the span's operation. - /// - /// For example, the name can be a qualified method name or a file name - /// and a line number where the operation is called. A best practice is to use - /// the same display name at the same call point in an application. - /// This makes it easier to correlate spans in different traces. - /// - /// This field is semantically required to be set to non-empty string. - /// Empty value is equivalent to an unknown span name. - /// - /// This field is required. #[prost(string, tag = "5")] pub name: ::prost::alloc::string::String, - /// Distinguishes between spans generated in a particular context. For example, - /// two spans with the same name may be distinguished using `CLIENT` (caller) - /// and `SERVER` (callee) to identify queueing latency associated with the span. #[prost(enumeration = "span::SpanKind", tag = "6")] pub kind: i32, - /// The start time of the span. On the client side, this is the time - /// kept by the local machine where the span execution starts. On the server side, this - /// is the time when the server's application handler starts running. - /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. - /// - /// This field is semantically required and it is expected that end_time >= start_time. #[prost(fixed64, tag = "7")] pub start_time_unix_nano: u64, - /// The end time of the span. On the client side, this is the time - /// kept by the local machine where the span execution ends. On the server side, this - /// is the time when the server application handler stops running. - /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. - /// - /// This field is semantically required and it is expected that end_time >= start_time. #[prost(fixed64, tag = "8")] pub end_time_unix_nano: u64, - /// A collection of key/value pairs. Note, global attributes - /// like server name can be set using the resource API. Examples of attributes: - /// - /// "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" - /// "/http/server_latency": 300 - /// "example.com/myattribute": true - /// "example.com/score": 10.239 - /// - /// Attribute keys MUST be unique (it is not allowed to have more than one - /// attribute with the same key). - /// The behavior of software that receives duplicated keys can be unpredictable. #[prost(message, repeated, tag = "9")] pub attributes: ::prost::alloc::vec::Vec, - /// The number of attributes that were discarded. Attributes - /// can be discarded because their keys are too long or because there are too many - /// attributes. If this value is 0, then no attributes were dropped. #[prost(uint32, tag = "10")] pub dropped_attributes_count: u32, - /// A collection of Event items. #[prost(message, repeated, tag = "11")] pub events: ::prost::alloc::vec::Vec, - /// The number of dropped events. If the value is 0, then no - /// events were dropped. #[prost(uint32, tag = "12")] pub dropped_events_count: u32, - /// A collection of Links, which are references from this span to a span - /// in the same or different trace. #[prost(message, repeated, tag = "13")] pub links: ::prost::alloc::vec::Vec, - /// The number of dropped links after the maximum size was - /// enforced. If this value is 0, then no links were dropped. #[prost(uint32, tag = "14")] pub dropped_links_count: u32, - /// An optional final status for this span. Semantically when Status isn't set, it means - /// span's status code is unset, i.e. assume STATUS_CODE_UNSET (code = 0). #[prost(message, optional, tag = "15")] pub status: ::core::option::Option, } /// Nested message and enum types in `Span`. pub mod span { - /// Event is a time-stamped annotation of the span, consisting of user-supplied - /// text description and key-value pairs. #[derive(Clone, PartialEq, ::prost::Message)] pub struct Event { - /// The time the event occurred. #[prost(fixed64, tag = "1")] pub time_unix_nano: u64, - /// The name of the event. - /// This field is semantically required to be set to non-empty string. #[prost(string, tag = "2")] pub name: ::prost::alloc::string::String, - /// A collection of attribute key/value pairs on the event. - /// Attribute keys MUST be unique (it is not allowed to have more than one - /// attribute with the same key). - /// The behavior of software that receives duplicated keys can be unpredictable. #[prost(message, repeated, tag = "3")] pub attributes: ::prost::alloc::vec::Vec< super::super::super::common::v1::KeyValue, >, - /// The number of dropped attributes. If the value is 0, - /// then no attributes were dropped. #[prost(uint32, tag = "4")] pub dropped_attributes_count: u32, } - /// A pointer from the current span to another span in the same trace or in a - /// different trace. For example, this can be used in batching operations, - /// where a single batch handler processes multiple requests from different - /// traces or when the handler receives a request from a different project. #[derive(Clone, PartialEq, ::prost::Message)] pub struct Link { - /// A unique identifier of a trace that this linked span is part of. The ID is a - /// 16-byte array. #[prost(bytes = "vec", tag = "1")] pub trace_id: ::prost::alloc::vec::Vec, - /// A unique identifier for the linked span. The ID is an 8-byte array. #[prost(bytes = "vec", tag = "2")] pub span_id: ::prost::alloc::vec::Vec, - /// The trace_state associated with the link. #[prost(string, tag = "3")] pub trace_state: ::prost::alloc::string::String, - /// A collection of attribute key/value pairs on the link. - /// Attribute keys MUST be unique (it is not allowed to have more than one - /// attribute with the same key). - /// The behavior of software that receives duplicated keys can be unpredictable. #[prost(message, repeated, tag = "4")] pub attributes: ::prost::alloc::vec::Vec< super::super::super::common::v1::KeyValue, >, - /// The number of dropped attributes. If the value is 0, - /// then no attributes were dropped. #[prost(uint32, tag = "5")] pub dropped_attributes_count: u32, - /// Flags, a bit field. - /// - /// Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace - /// Context specification. To read the 8-bit W3C trace flag, use - /// `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. - /// - /// See for the flag definitions. - /// - /// Bits 8 and 9 represent the 3 states of whether the link is remote. - /// The states are (unknown, is not remote, is remote). - /// To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. - /// To read whether the link is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. - /// - /// Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. - /// When creating new spans, bits 10-31 (most-significant 22-bits) MUST be zero. - /// - /// \[Optional\]. #[prost(fixed32, tag = "6")] pub flags: u32, } - /// SpanKind is the type of span. Can be used to specify additional relationships between spans - /// in addition to a parent/child relationship. #[derive( Clone, Copy, @@ -275,25 +105,11 @@ pub mod span { )] #[repr(i32)] pub enum SpanKind { - /// Unspecified. Do NOT use as default. - /// Implementations MAY assume SpanKind to be INTERNAL when receiving UNSPECIFIED. Unspecified = 0, - /// Indicates that the span represents an internal operation within an application, - /// as opposed to an operation happening at the boundaries. Default value. Internal = 1, - /// Indicates that the span covers server-side handling of an RPC or other - /// remote network request. Server = 2, - /// Indicates that the span describes a request to some remote service. Client = 3, - /// Indicates that the span describes a producer sending a message to a broker. - /// Unlike CLIENT and SERVER, there is often no direct critical path latency relationship - /// between producer and consumer spans. A PRODUCER span ends when the message was accepted - /// by the broker while the logical processing of the message might span a much longer time. Producer = 4, - /// Indicates that the span describes consumer receiving a message from a broker. - /// Like the PRODUCER kind, there is often no direct critical path latency relationship - /// between producer and consumer spans. Consumer = 5, } impl SpanKind { @@ -325,21 +141,15 @@ pub mod span { } } } -/// The Status type defines a logical error model that is suitable for different -/// programming environments, including REST APIs and RPC APIs. #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Status { - /// A developer-facing human readable error message. #[prost(string, tag = "2")] pub message: ::prost::alloc::string::String, - /// The status code. #[prost(enumeration = "status::StatusCode", tag = "3")] pub code: i32, } /// Nested message and enum types in `Status`. pub mod status { - /// For the semantics of status codes see - /// #[derive( Clone, Copy, @@ -353,12 +163,8 @@ pub mod status { )] #[repr(i32)] pub enum StatusCode { - /// The default status. Unset = 0, - /// The Span has been validated by an Application developer or Operator to - /// have completed successfully. Ok = 1, - /// The Span contains an error. Error = 2, } impl StatusCode { @@ -384,31 +190,11 @@ pub mod status { } } } -/// SpanFlags represents constants used to interpret the -/// Span.flags field, which is protobuf 'fixed32' type and is to -/// be used as bit-fields. Each non-zero value defined in this enum is -/// a bit-mask. To extract the bit-field, for example, use an -/// expression like: -/// -/// (span.flags & SPAN_FLAGS_TRACE_FLAGS_MASK) -/// -/// See for the flag definitions. -/// -/// Note that Span flags were introduced in version 1.1 of the -/// OpenTelemetry protocol. Older Span producers do not set this -/// field, consequently consumers should not rely on the absence of a -/// particular flag bit to indicate the presence of a particular feature. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum SpanFlags { - /// The zero value for the enum. Should not be used for comparisons. - /// Instead use bitwise "and" with the appropriate mask as shown above. DoNotUse = 0, - /// Bits 0-7 are used for trace flags. TraceFlagsMask = 255, - /// Bits 8 and 9 are used to indicate that the parent span or link span is remote. - /// Bit 8 (`HAS_IS_REMOTE`) indicates whether the value is known. - /// Bit 9 (`IS_REMOTE`) indicates whether the span or link is remote. ContextHasIsRemoteMask = 256, ContextIsRemoteMask = 512, } From 664f16fb8c1c5578213cdc8d9c84268b9d9aed56 Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 15:27:32 -0400 Subject: [PATCH 15/17] docs(data-pipeline): clarify parse_u64/kind fallbacks and unreachable Grpc content-type arm --- libdd-data-pipeline/src/otlp/exporter.rs | 2 ++ libdd-trace-utils/src/otlp_encoder/proto_convert.rs | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/libdd-data-pipeline/src/otlp/exporter.rs b/libdd-data-pipeline/src/otlp/exporter.rs index 929e42fa10..ba7ccef060 100644 --- a/libdd-data-pipeline/src/otlp/exporter.rs +++ b/libdd-data-pipeline/src/otlp/exporter.rs @@ -43,6 +43,8 @@ pub async fn send_otlp_traces_http( ..Endpoint::default() }; + // `Grpc` is rejected earlier in `send_otlp_traces_inner` and never reaches this function, so it + // is grouped with the JSON content-type here only to keep the match exhaustive. let content_type = match config.protocol { crate::otlp::config::OtlpProtocol::HttpProtobuf => { libdd_common::header::APPLICATION_PROTOBUF diff --git a/libdd-trace-utils/src/otlp_encoder/proto_convert.rs b/libdd-trace-utils/src/otlp_encoder/proto_convert.rs index a3658981d1..daa68565a5 100644 --- a/libdd-trace-utils/src/otlp_encoder/proto_convert.rs +++ b/libdd-trace-utils/src/otlp_encoder/proto_convert.rs @@ -48,6 +48,9 @@ fn hex_nibble(b: u8) -> Option { } } +/// Parse a decimal timestamp string into `u64`. `mapper.rs` always emits these from `u64`/`i64` +/// fields via `format!`, so a parse failure can only mean a mapper bug; we fall back to 0 rather +/// than panicking (FFI reliability), matching the zero-fallback policy of `hex_to_bytes`. fn parse_u64(s: &str) -> u64 { s.parse().unwrap_or(0) } @@ -127,6 +130,8 @@ fn span(s: &j::OtlpSpan) -> ProtoSpan { .unwrap_or_default(), flags: s.flags.unwrap_or(0), name: s.name.clone(), + // `kind` is a prost open enum (stored as i32); the mapper produces valid SpanKind values, + // and unknown values are passed through unchanged per OTLP open-enum semantics. kind: s.kind, start_time_unix_nano: parse_u64(&s.start_time_unix_nano), end_time_unix_nano: parse_u64(&s.end_time_unix_nano), From 9bc5cf297021fe125e169c1fecd38b7a3766efae Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 17:28:22 -0400 Subject: [PATCH 16/17] chore: drop in-repo planning docs (moved to chonk) The OTLP design spec and implementation plan are linked from the PR description (internal chonk) rather than committed to the repo. Co-Authored-By: Claude Opus 4.8 --- .../2026-06-12-otlp-http-protobuf-export.md | 1199 ----------------- ...-06-12-otlp-http-protobuf-export-design.md | 251 ---- 2 files changed, 1450 deletions(-) delete mode 100644 docs/superpowers/plans/2026-06-12-otlp-http-protobuf-export.md delete mode 100644 docs/superpowers/specs/2026-06-12-otlp-http-protobuf-export-design.md diff --git a/docs/superpowers/plans/2026-06-12-otlp-http-protobuf-export.md b/docs/superpowers/plans/2026-06-12-otlp-http-protobuf-export.md deleted file mode 100644 index 90e865bb27..0000000000 --- a/docs/superpowers/plans/2026-06-12-otlp-http-protobuf-export.md +++ /dev/null @@ -1,1199 +0,0 @@ -# OTLP HTTP/protobuf trace export — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add OTLP HTTP/protobuf as a second trace-export encoding alongside HTTP/JSON in libdatadog, selectable via the OTel-standard protocol values, and wire it through dd-trace-py with end-to-end validation. - -**Architecture:** Vendor the OTLP `trace` + `collector/trace` protos into `libdd-trace-protobuf` and generate prost types (zero new runtime deps). Keep the existing hand-rolled serde JSON path untouched. The semantic DD-span→OTLP mapping runs once and produces the serde types; a mechanical `From<&serde_types>` converter produces the prost types for protobuf. The exporter selects encoder + content-type from `OtlpProtocol`. dd-trace-py gains a `set_otlp_protocol` binding and passes its already-parsed `TRACES_PROTOCOL` through. - -**Tech Stack:** Rust (prost 0.14, prost-build, protoc-bin-vendored, serde_json, httpmock), C FFI (cbindgen), Python/PyO3 (setuptools-rust), system-tests, sdk-backend-verify. - -**Spec:** `docs/superpowers/specs/2026-06-12-otlp-http-protobuf-export-design.md` - -**Refinement vs spec:** The spec proposed gating the protobuf encoder behind an `otlp-protobuf` cargo feature. During planning we confirmed the generated OTLP types live in `libdd-trace-protobuf` (already a non-optional dep of `libdd-trace-utils`) and, matching the existing OTLP common/resource pattern, are compiled unconditionally. A feature gate would only guard the small converter module for negligible benefit, so this plan drops the gate (YAGNI). No new runtime dependency is introduced either way. - -**Worktrees:** -- libdatadog: `/Users/brian.marks/go/src/github.com/DataDog/libdatadog-otlp-http-protobuf-export` (branch `brian.marks/otlp-http-protobuf-export`) — already created. -- dd-trace-py: create at execution time (Phase 6). - ---- - -## Phase 0 — Footprint spike (go/no-go gate) - -### Task 0: Confirm vendored prost types add no heavy dependencies - -**Files:** none (investigation). - -- [ ] **Step 1: Record the OTel proto version already vendored** - -Run: -```bash -cd /Users/brian.marks/go/src/github.com/DataDog/libdatadog-otlp-http-protobuf-export -head -20 libdd-trace-protobuf/src/pb/opentelemetry/proto/common/v1/common.proto -``` -Expected: a header/comment indicating the upstream opentelemetry-proto version (e.g. a release tag or proto package version). Note this version — Phase 1 vendors `trace.proto` + `trace_service.proto` from the **same** release for import compatibility. - -- [ ] **Step 2: Confirm prost is already the protobuf toolchain (no new runtime crate needed)** - -Run: -```bash -grep -n 'prost' libdd-trace-protobuf/Cargo.toml libdd-trace-utils/Cargo.toml -``` -Expected: `prost = "0.14.x"` present in both; `prost-build` + `protoc-bin-vendored` present in `libdd-trace-protobuf` under `[build-dependencies]` behind `generate-protobuf`. Conclusion: vendoring adds only generated structs, no new external runtime crate. - -- [ ] **Step 3: Gate decision** - -If Steps 1–2 hold (they should, per the spec's prior investigation), proceed. If a new heavy crate would be required, STOP and revisit the spec's decision 1. - ---- - -## Phase 1 — Generate OTLP trace + collector prost types (`libdd-trace-protobuf`) - -### Task 1: Vendor the OTLP trace + collector protos and generate prost types - -**Files:** -- Create: `libdd-trace-protobuf/src/pb/opentelemetry/proto/trace/v1/trace.proto` -- Create: `libdd-trace-protobuf/src/pb/opentelemetry/proto/collector/trace/v1/trace_service.proto` -- Modify: `libdd-trace-protobuf/build.rs` (compile list + license prepend) -- Create (generated, committed): `libdd-trace-protobuf/src/opentelemetry.proto.trace.v1.rs`, `libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs` - -- [ ] **Step 1: Vendor the two proto files from the matching opentelemetry-proto release** - -Use the same release tag noted in Task 0. From the opentelemetry-proto repo, copy verbatim: -- `opentelemetry/proto/trace/v1/trace.proto` → `libdd-trace-protobuf/src/pb/opentelemetry/proto/trace/v1/trace.proto` -- `opentelemetry/proto/collector/trace/v1/trace_service.proto` → `libdd-trace-protobuf/src/pb/opentelemetry/proto/collector/trace/v1/trace_service.proto` - -```bash -mkdir -p libdd-trace-protobuf/src/pb/opentelemetry/proto/trace/v1 -mkdir -p libdd-trace-protobuf/src/pb/opentelemetry/proto/collector/trace/v1 -TAG= # e.g. v1.5.0 -BASE="https://raw.githubusercontent.com/open-telemetry/opentelemetry-proto/$TAG/opentelemetry/proto" -curl -fsSL "$BASE/trace/v1/trace.proto" -o libdd-trace-protobuf/src/pb/opentelemetry/proto/trace/v1/trace.proto -curl -fsSL "$BASE/collector/trace/v1/trace_service.proto" -o libdd-trace-protobuf/src/pb/opentelemetry/proto/collector/trace/v1/trace_service.proto -``` -Expected: both files saved. `trace.proto` imports `opentelemetry/proto/common/v1/common.proto` and `.../resource/v1/resource.proto` (already vendored). `trace_service.proto` imports `.../trace/v1/trace.proto` and defines `ExportTraceServiceRequest`/`ExportTraceServiceResponse`. - -- [ ] **Step 2: Add both protos to the compile list in `build.rs`** - -In `libdd-trace-protobuf/build.rs`, extend the `compile_protos(&[ ... ], &["src/pb/"])` array (currently ending at `"src/pb/idx/span.proto"`): - -```rust - &[ - "src/pb/agent_payload.proto", - "src/pb/tracer_payload.proto", - "src/pb/span.proto", - "src/pb/stats.proto", - "src/pb/remoteconfig.proto", - "src/pb/opentelemetry/proto/common/v1/process_context.proto", - "src/pb/opentelemetry/proto/trace/v1/trace.proto", - "src/pb/opentelemetry/proto/collector/trace/v1/trace_service.proto", - "src/pb/idx/tracer_payload.proto", - "src/pb/idx/span.proto", - ], -``` - -- [ ] **Step 3: Prepend the OTel license header to the new generated files** - -In `build.rs`, next to the existing `prepend_to_file(otel_license, ...resource.v1.rs)` / `...common.v1.rs` calls, add: - -```rust - prepend_to_file( - otel_license, - &output_path.join("opentelemetry.proto.trace.v1.rs"), - ); - prepend_to_file( - otel_license, - &output_path.join("opentelemetry.proto.collector.trace.v1.rs"), - ); -``` - -- [ ] **Step 4: Regenerate the committed Rust types** - -Run: -```bash -cargo build -p libdd-trace-protobuf --features generate-protobuf -``` -Expected: build succeeds; new files `libdd-trace-protobuf/src/opentelemetry.proto.trace.v1.rs` and `libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs` appear, and `libdd-trace-protobuf/src/_includes.rs` now references the `opentelemetry::proto::trace::v1` and `opentelemetry::proto::collector::trace::v1` modules. - -- [ ] **Step 5: Verify the generated type path compiles and is reachable** - -Run: -```bash -cargo build -p libdd-trace-protobuf -``` -Then confirm the symbol path with a throwaway check: -```bash -grep -rn "ExportTraceServiceRequest" libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs | head -``` -Expected: `pub struct ExportTraceServiceRequest` present. Its module path is `libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest`, with span/resource types under `opentelemetry::proto::trace::v1` and `...::common::v1` / `...::resource::v1`. - -- [ ] **Step 6: Commit** - -```bash -git add libdd-trace-protobuf/src/pb/opentelemetry libdd-trace-protobuf/build.rs \ - libdd-trace-protobuf/src/opentelemetry.proto.trace.v1.rs \ - libdd-trace-protobuf/src/opentelemetry.proto.collector.trace.v1.rs \ - libdd-trace-protobuf/src/_includes.rs -git commit -m "feat(trace-protobuf): vendor + generate OTLP trace/collector prost types" -``` - ---- - -## Phase 2 — Converter + protobuf encoder (`libdd-trace-utils::otlp_encoder`) - -Module paths below assume the generated types are re-exported as -`libdd_trace_protobuf::opentelemetry::proto::{trace::v1 as otlp_trace, common::v1 as otlp_common, resource::v1 as otlp_resource, collector::trace::v1 as otlp_collector}`. Confirm exact paths from Task 1 Step 5 and adjust the `use` lines if the generated module nesting differs. - -### Task 2: serde→prost converter for the OTLP request - -**Files:** -- Create: `libdd-trace-utils/src/otlp_encoder/proto_convert.rs` -- Modify: `libdd-trace-utils/src/otlp_encoder/mod.rs` (declare module + re-export encoders) -- Test: inline `#[cfg(test)]` in `proto_convert.rs` - -- [ ] **Step 1: Write the failing test for the converter** - -Create `libdd-trace-utils/src/otlp_encoder/proto_convert.rs` with only the test module first: - -```rust -// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ -// SPDX-License-Identifier: Apache-2.0 - -#[cfg(test)] -mod tests { - use crate::otlp_encoder::{map_traces_to_otlp, OtlpResourceInfo}; - use crate::span::BytesData; - use crate::span::v04::Span; - use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest as ProtoReq; - - #[test] - fn converts_ids_and_attributes_to_proto() { - let resource_info = OtlpResourceInfo { - service: "svc".to_string(), - ..Default::default() - }; - let mut span: Span = Span { - trace_id: 0xD269B633813FC60C_u128, - span_id: 0xEEE19B7EC3C1B174, - parent_id: 0xEEE19B7EC3C1B173, - name: libdd_tinybytes::BytesString::from_static("op"), - resource: libdd_tinybytes::BytesString::from_static("res"), - r#type: libdd_tinybytes::BytesString::from_static("web"), - start: 1544712660000000000, - duration: 1000000000, - error: 0, - ..Default::default() - }; - span.metrics - .insert(libdd_tinybytes::BytesString::from_static("count"), 42.0); - - let serde_req = map_traces_to_otlp(vec![vec![span]], &resource_info); - let proto: ProtoReq = (&serde_req).into(); - - let rs = &proto.resource_spans[0]; - let sp = &rs.scope_spans[0].spans[0]; - // trace_id: 16 bytes, big-endian, high 64 bits zero (no _dd.p.tid) - assert_eq!( - sp.trace_id, - vec![0, 0, 0, 0, 0, 0, 0, 0, 0xD2, 0x69, 0xB6, 0x33, 0x81, 0x3F, 0xC6, 0x0C] - ); - assert_eq!(sp.span_id, vec![0xEE, 0xE1, 0x9B, 0x7E, 0xC3, 0xC1, 0xB1, 0x74]); - assert_eq!(sp.parent_span_id, vec![0xEE, 0xE1, 0x9B, 0x7E, 0xC3, 0xC1, 0xB1, 0x73]); - assert_eq!(sp.name, "res"); - assert_eq!(sp.start_time_unix_nano, 1544712660000000000); - assert_eq!(sp.end_time_unix_nano, 1544712661000000000); - // count metric -> int attribute - let count = sp - .attributes - .iter() - .find(|kv| kv.key == "count") - .expect("count attr"); - use libdd_trace_protobuf::opentelemetry::proto::common::v1::any_value::Value; - assert!(matches!(count.value.as_ref().unwrap().value, Some(Value::IntValue(42)))); - } -} -``` - -- [ ] **Step 2: Run the test to verify it fails to compile (no `From` impl yet)** - -Run: -```bash -cargo test -p libdd-trace-utils otlp_encoder::proto_convert -- --nocapture -``` -Expected: compile error — `From<&ExportTraceServiceRequest>` not implemented / trait bound not satisfied. - -- [ ] **Step 3: Implement the converter** - -Prepend the implementation above the test module in `proto_convert.rs`. Use the generated module paths confirmed in Task 1 Step 5: - -```rust -//! Converts the hand-rolled serde OTLP request (the JSON wire model) into the generated -//! prost types for binary (HTTP/protobuf) export. The semantic DD-span -> OTLP mapping already -//! happened in `mapper.rs`; this is a purely structural translation. - -use crate::otlp_encoder::json_types as j; -use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest as ProtoReq; -use libdd_trace_protobuf::opentelemetry::proto::common::v1::{ - any_value::Value as ProtoValue, AnyValue as ProtoAnyValue, ArrayValue as ProtoArrayValue, - InstrumentationScope as ProtoScope, KeyValue as ProtoKeyValue, -}; -use libdd_trace_protobuf::opentelemetry::proto::resource::v1::Resource as ProtoResource; -use libdd_trace_protobuf::opentelemetry::proto::trace::v1::{ - span::{Event as ProtoEvent, Link as ProtoLink}, - status::StatusCode as ProtoStatusCode, - ResourceSpans as ProtoResourceSpans, ScopeSpans as ProtoScopeSpans, Span as ProtoSpan, - Status as ProtoStatus, -}; - -/// Decode a fixed-width lowercase hex string into a byte vector. The mapper always produces -/// well-formed hex of the expected width; on the unexpected event of a malformed value we fall -/// back to an all-zero buffer of `len` bytes rather than panicking (FFI reliability). -fn hex_to_bytes(s: &str, len: usize) -> Vec { - let mut out = Vec::with_capacity(len); - let bytes = s.as_bytes(); - if bytes.len() == len * 2 { - let mut i = 0; - while i < bytes.len() { - match (hex_nibble(bytes[i]), hex_nibble(bytes[i + 1])) { - (Some(hi), Some(lo)) => out.push((hi << 4) | lo), - _ => return vec![0u8; len], - } - i += 2; - } - out - } else { - vec![0u8; len] - } -} - -fn hex_nibble(b: u8) -> Option { - match b { - b'0'..=b'9' => Some(b - b'0'), - b'a'..=b'f' => Some(b - b'a' + 10), - b'A'..=b'F' => Some(b - b'A' + 10), - _ => None, - } -} - -fn parse_u64(s: &str) -> u64 { - s.parse().unwrap_or(0) -} - -impl From<&j::AnyValue> for ProtoAnyValue { - fn from(v: &j::AnyValue) -> Self { - let value = match v { - j::AnyValue::StringValue(s) => ProtoValue::StringValue(s.clone()), - j::AnyValue::BoolValue(b) => ProtoValue::BoolValue(*b), - j::AnyValue::IntValue(i) => ProtoValue::IntValue(*i), - j::AnyValue::DoubleValue(d) => ProtoValue::DoubleValue(*d), - j::AnyValue::BytesValue(b) => ProtoValue::BytesValue(b.clone()), - j::AnyValue::ArrayValue(a) => ProtoValue::ArrayValue(ProtoArrayValue { - values: a.values.iter().map(ProtoAnyValue::from).collect(), - }), - }; - ProtoAnyValue { value: Some(value) } - } -} - -fn kv(k: &j::KeyValue) -> ProtoKeyValue { - ProtoKeyValue { - key: k.key.clone(), - value: Some(ProtoAnyValue::from(&k.value)), - } -} - -impl From<&j::ExportTraceServiceRequest> for ProtoReq { - fn from(req: &j::ExportTraceServiceRequest) -> Self { - ProtoReq { - resource_spans: req.resource_spans.iter().map(resource_spans).collect(), - } - } -} - -fn resource_spans(rs: &j::ResourceSpans) -> ProtoResourceSpans { - ProtoResourceSpans { - resource: rs.resource.as_ref().map(|r| ProtoResource { - attributes: r.attributes.iter().map(kv).collect(), - dropped_attributes_count: 0, - }), - scope_spans: rs.scope_spans.iter().map(scope_spans).collect(), - schema_url: String::new(), - } -} - -fn scope_spans(ss: &j::ScopeSpans) -> ProtoScopeSpans { - ProtoScopeSpans { - scope: ss.scope.as_ref().map(|s| ProtoScope { - name: s.name.clone().unwrap_or_default(), - version: s.version.clone().unwrap_or_default(), - attributes: Vec::new(), - dropped_attributes_count: 0, - }), - spans: ss.spans.iter().map(span).collect(), - schema_url: ss.schema_url.clone().unwrap_or_default(), - } -} - -fn span(s: &j::OtlpSpan) -> ProtoSpan { - ProtoSpan { - trace_id: hex_to_bytes(&s.trace_id, 16), - span_id: hex_to_bytes(&s.span_id, 8), - trace_state: s.trace_state.clone().unwrap_or_default(), - parent_span_id: s - .parent_span_id - .as_ref() - .map(|p| hex_to_bytes(p, 8)) - .unwrap_or_default(), - flags: s.flags.unwrap_or(0), - name: s.name.clone(), - kind: s.kind, - start_time_unix_nano: parse_u64(&s.start_time_unix_nano), - end_time_unix_nano: parse_u64(&s.end_time_unix_nano), - attributes: s.attributes.iter().map(kv).collect(), - dropped_attributes_count: s.dropped_attributes_count.unwrap_or(0), - events: s.events.iter().map(event).collect(), - dropped_events_count: s.dropped_events_count.unwrap_or(0), - links: s.links.iter().map(link).collect(), - dropped_links_count: 0, - status: Some(ProtoStatus { - message: s.status.message.clone().unwrap_or_default(), - code: status_code(s.status.code), - }), - } -} - -fn status_code(code: i32) -> i32 { - // Mirror j::status_code constants onto the generated enum's i32 values. - match code { - c if c == j::status_code::OK => ProtoStatusCode::Ok as i32, - c if c == j::status_code::ERROR => ProtoStatusCode::Error as i32, - _ => ProtoStatusCode::Unset as i32, - } -} - -fn link(l: &j::OtlpSpanLink) -> ProtoLink { - ProtoLink { - trace_id: hex_to_bytes(&l.trace_id, 16), - span_id: hex_to_bytes(&l.span_id, 8), - trace_state: l.trace_state.clone().unwrap_or_default(), - attributes: l.attributes.iter().map(kv).collect(), - dropped_attributes_count: l.dropped_attributes_count.unwrap_or(0), - flags: 0, - } -} - -fn event(e: &j::OtlpSpanEvent) -> ProtoEvent { - ProtoEvent { - time_unix_nano: parse_u64(&e.time_unix_nano), - name: e.name.clone(), - attributes: e.attributes.iter().map(kv).collect(), - dropped_attributes_count: e.dropped_attributes_count.unwrap_or(0), - } -} -``` - -> Note: field names/struct shapes above are the standard prost OTLP output, but prost can name nested types differently across versions. After generation (Task 1), open `opentelemetry.proto.trace.v1.rs` and reconcile any field names (`dropped_links_count`, `flags`, the `span::{Event, Link}` / `status::StatusCode` nesting) with the generated source before finishing this task. - -- [ ] **Step 4: Declare the module in `mod.rs`** - -In `libdd-trace-utils/src/otlp_encoder/mod.rs`, add under the existing `pub mod mapper;`: -```rust -pub mod proto_convert; -``` - -- [ ] **Step 5: Run the converter test to verify it passes** - -Run: -```bash -cargo test -p libdd-trace-utils otlp_encoder::proto_convert -- --nocapture -``` -Expected: PASS. If field-name mismatches appear, fix per the note in Step 3, then re-run. - -- [ ] **Step 6: Commit** - -```bash -git add libdd-trace-utils/src/otlp_encoder/proto_convert.rs libdd-trace-utils/src/otlp_encoder/mod.rs -git commit -m "feat(trace-utils): add serde->prost OTLP converter" -``` - -### Task 3: Public encoders (`encode_otlp_json`, `encode_otlp_protobuf`) + parity test - -**Files:** -- Modify: `libdd-trace-utils/src/otlp_encoder/mod.rs` -- Test: inline `#[cfg(test)]` in `mod.rs` - -- [ ] **Step 1: Write the failing parity test** - -Add to `libdd-trace-utils/src/otlp_encoder/mod.rs` a test module: - -```rust -#[cfg(test)] -mod encode_tests { - use super::*; - use crate::span::BytesData; - use crate::span::v04::Span; - use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest as ProtoReq; - use prost::Message; - - fn sample() -> ExportTraceServiceRequest { - let resource_info = OtlpResourceInfo { service: "svc".to_string(), ..Default::default() }; - let span: Span = Span { - trace_id: 0xD269B633813FC60C_u128, - span_id: 0xEEE19B7EC3C1B174, - name: libdd_tinybytes::BytesString::from_static("op"), - resource: libdd_tinybytes::BytesString::from_static("res"), - start: 1, duration: 2, ..Default::default() - }; - map_traces_to_otlp(vec![vec![span]], &resource_info) - } - - #[test] - fn json_and_protobuf_carry_same_span() { - let req = sample(); - let json = encode_otlp_json(&req).unwrap(); - let pb = encode_otlp_protobuf(&req); - - let json_v: serde_json::Value = serde_json::from_slice(&json).unwrap(); - let json_name = json_v["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["name"] - .as_str().unwrap().to_string(); - - let proto = ProtoReq::decode(pb.as_slice()).unwrap(); - let proto_name = proto.resource_spans[0].scope_spans[0].spans[0].name.clone(); - - assert_eq!(json_name, "res"); - assert_eq!(proto_name, "res"); - // Span id round-trips identically: JSON hex vs proto bytes. - let json_sid = json_v["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["spanId"] - .as_str().unwrap().to_string(); - let proto_sid = &proto.resource_spans[0].scope_spans[0].spans[0].span_id; - assert_eq!(json_sid, hex::encode(proto_sid)); - } -} -``` - -Add `hex = "0.4"` to `libdd-trace-utils` `[dev-dependencies]` if not already present (used only in tests). - -- [ ] **Step 2: Run to verify it fails (encoders not defined)** - -Run: -```bash -cargo test -p libdd-trace-utils otlp_encoder::encode_tests -- --nocapture -``` -Expected: compile error — `encode_otlp_json` / `encode_otlp_protobuf` not found. - -- [ ] **Step 3: Implement the encoders** - -Add to `libdd-trace-utils/src/otlp_encoder/mod.rs` (after the `pub use mapper::map_traces_to_otlp;` line): - -```rust -use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest as ProtoExportTraceServiceRequest; -use prost::Message; - -pub use json_types::ExportTraceServiceRequest; - -/// Serialize an OTLP request to the HTTP/JSON wire format. -pub fn encode_otlp_json( - req: &ExportTraceServiceRequest, -) -> serde_json::Result> { - serde_json::to_vec(req) -} - -/// Serialize an OTLP request to the HTTP/protobuf wire format. -pub fn encode_otlp_protobuf(req: &ExportTraceServiceRequest) -> Vec { - let proto: ProtoExportTraceServiceRequest = req.into(); - proto.encode_to_vec() -} -``` - -- [ ] **Step 4: Run to verify it passes** - -Run: -```bash -cargo test -p libdd-trace-utils otlp_encoder:: -- --nocapture -``` -Expected: PASS (parity test + converter test + existing mapper tests all green). - -- [ ] **Step 5: Commit** - -```bash -git add libdd-trace-utils/src/otlp_encoder/mod.rs libdd-trace-utils/Cargo.toml -git commit -m "feat(trace-utils): add encode_otlp_json/encode_otlp_protobuf" -``` - ---- - -## Phase 3 — Protocol selection + dispatch (`libdd-data-pipeline`) - -### Task 4: Make `OtlpProtocol` public + `FromStr` - -**Files:** -- Modify: `libdd-data-pipeline/src/otlp/config.rs` -- Test: inline `#[cfg(test)]` in `config.rs` - -- [ ] **Step 1: Write the failing test** - -Add to `libdd-data-pipeline/src/otlp/config.rs`: - -```rust -#[cfg(test)] -mod tests { - use super::*; - use std::str::FromStr; - - #[test] - fn protocol_from_str() { - assert_eq!(OtlpProtocol::from_str("http/json").unwrap(), OtlpProtocol::HttpJson); - assert_eq!(OtlpProtocol::from_str("http/protobuf").unwrap(), OtlpProtocol::HttpProtobuf); - assert_eq!(OtlpProtocol::from_str("grpc").unwrap(), OtlpProtocol::Grpc); - assert!(OtlpProtocol::from_str("nonsense").is_err()); - } -} -``` - -- [ ] **Step 2: Run to verify it fails** - -Run: `cargo test -p libdd-data-pipeline otlp::config -- --nocapture` -Expected: compile error — `from_str` not implemented; `OtlpProtocol` not public. - -- [ ] **Step 3: Implement** - -In `libdd-data-pipeline/src/otlp/config.rs` change the enum visibility and remove the dead-code allow on `HttpProtobuf`: - -```rust -/// OTLP trace export protocol. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum OtlpProtocol { - /// HTTP with JSON body (Content-Type: application/json). Default for HTTP. - #[default] - HttpJson, - /// HTTP with protobuf body (Content-Type: application/x-protobuf). - HttpProtobuf, - /// gRPC. (Not supported yet) - #[allow(dead_code)] - Grpc, -} - -impl std::str::FromStr for OtlpProtocol { - type Err = String; - fn from_str(s: &str) -> Result { - match s { - "http/json" => Ok(OtlpProtocol::HttpJson), - "http/protobuf" => Ok(OtlpProtocol::HttpProtobuf), - "grpc" => Ok(OtlpProtocol::Grpc), - other => Err(format!("unknown OTLP protocol: {other}")), - } - } -} -``` -Also change `protocol: OtlpProtocol` field on `OtlpTraceConfig` from `pub(crate)` to `pub` and drop its `#[allow(dead_code)]`. - -- [ ] **Step 4: Run to verify it passes** - -Run: `cargo test -p libdd-data-pipeline otlp::config -- --nocapture` -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add libdd-data-pipeline/src/otlp/config.rs -git commit -m "feat(data-pipeline): make OtlpProtocol public with FromStr" -``` - -### Task 5: Content-type by protocol in the transport - -**Files:** -- Modify: `libdd-data-pipeline/src/otlp/exporter.rs` - -- [ ] **Step 1: Update `send_otlp_traces_http` to choose content-type from protocol** - -In `libdd-data-pipeline/src/otlp/exporter.rs`, rename the `json_body: Vec` parameter to `body: Vec`, pass it to `send_with_retry` instead of `json_body`, and replace the hardcoded content-type insert: - -```rust - let content_type = match config.protocol { - crate::otlp::config::OtlpProtocol::HttpProtobuf => libdd_common::header::APPLICATION_PROTOBUF, - _ => libdd_common::header::APPLICATION_JSON, - }; - let mut headers = config.headers.clone(); - headers.insert(http::header::CONTENT_TYPE, content_type); -``` - -- [ ] **Step 2: Verify it compiles** - -Run: `cargo check -p libdd-data-pipeline` -Expected: success (the caller still passes JSON bytes; updated in Task 6). - -- [ ] **Step 3: Commit** - -```bash -git add libdd-data-pipeline/src/otlp/exporter.rs -git commit -m "feat(data-pipeline): set OTLP content-type from protocol" -``` - -### Task 6: Encoder dispatch in the send path - -**Files:** -- Modify: `libdd-data-pipeline/src/trace_exporter/mod.rs` (`send_otlp_traces_inner`, ~line 548) -- Modify: `libdd-data-pipeline/src/trace_exporter/mod.rs` imports (~line 18) - -- [ ] **Step 1: Replace the hardcoded JSON serialization with protocol dispatch** - -In `send_otlp_traces_inner`, replace the `serde_json::to_vec(&request)` block with: - -```rust - let request = map_traces_to_otlp(traces, &resource_info); - let body = match config.protocol { - OtlpProtocol::HttpJson => { - libdd_trace_utils::otlp_encoder::encode_otlp_json(&request).map_err(|e| { - error!("OTLP JSON serialization error: {e}"); - TraceExporterError::Internal(InternalErrorKind::InvalidWorkerState(e.to_string())) - })? - } - OtlpProtocol::HttpProtobuf => { - libdd_trace_utils::otlp_encoder::encode_otlp_protobuf(&request) - } - OtlpProtocol::Grpc => { - return Err(TraceExporterError::Internal(InternalErrorKind::InvalidWorkerState( - "OTLP gRPC export is not supported".to_string(), - ))); - } - }; - send_otlp_traces_http( - &self.capabilities, - config, - self.endpoint.test_token.as_deref(), - body, - ) - .await?; -``` - -Add `OtlpProtocol` to the `use crate::otlp::{...}` import line at the top of the file. - -- [ ] **Step 2: Verify the workspace builds** - -Run: `cargo check -p libdd-data-pipeline` -Expected: success. - -- [ ] **Step 3: Add a protobuf export integration test** - -Create `libdd-data-pipeline/tests/test_trace_exporter_otlp_protobuf_export.rs`: - -```rust -// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ -// SPDX-License-Identifier: Apache-2.0 -#[cfg(test)] -mod otlp_protobuf_tests { - use libdd_capabilities_impl::NativeCapabilities; - use libdd_data_pipeline::trace_exporter::TraceExporterBuilder; - use libdd_trace_utils::test_utils::create_test_json_span; - use libdd_trace_protobuf::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest; - use prost::Message; - use serde_json::json; - use tokio::task; - - #[cfg_attr(miri, ignore)] - #[tokio::test] - async fn otlp_protobuf_export_sends_decodable_payload() { - use httpmock::MockServer; - let server = MockServer::start_async().await; - let mut mock = server - .mock_async(|when, then| { - when.method("POST") - .path("/v1/traces") - .header("content-type", "application/x-protobuf"); - then.status(200).body(""); - }) - .await; - - let endpoint = format!("http://localhost:{}/v1/traces", server.port()); - let task_result = task::spawn_blocking(move || { - let mut builder = TraceExporterBuilder::default(); - builder - .set_otlp_endpoint(&endpoint) - .set_otlp_protocol(libdd_data_pipeline::otlp::config::OtlpProtocol::HttpProtobuf) - .set_language("test-lang") - .set_tracer_version("1.0") - .set_env("test_env") - .set_service("test"); - let exporter = builder.build::().expect("build"); - let mut span = create_test_json_span(1234, 12342, 12341, 1, false); - span["name"] = json!("pb_span"); - let data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); - exporter.send(data.as_ref()).expect("send ok"); - }) - .await; - assert!(task_result.is_ok()); - assert_eq!(mock.calls_async().await, 1); - - // Decode the most recent request body as protobuf to prove wire correctness. - let received = mock.received_requests_async().await.unwrap(); - let body = &received[0].body; - let req = ExportTraceServiceRequest::decode(body.as_slice()).expect("valid protobuf"); - let svc = req.resource_spans[0] - .resource - .as_ref() - .unwrap() - .attributes - .iter() - .find(|kv| kv.key == "service.name") - .unwrap(); - use libdd_trace_protobuf::opentelemetry::proto::common::v1::any_value::Value; - assert!(matches!(svc.value.as_ref().unwrap().value, Some(Value::StringValue(ref s)) if s == "test")); - mock.delete(); - } -} -``` - -> If `received_requests_async` / body access differs in the pinned httpmock version, mirror the body-capture approach already used elsewhere in `libdd-data-pipeline/tests/`. Confirm `set_otlp_protocol` exists on the builder (Task 7) before running — order Task 7 before this step if executing strictly sequentially. - -- [ ] **Step 4: Run the new test (after Task 7's builder method exists)** - -Run: `cargo nextest run -p libdd-data-pipeline otlp_protobuf` -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add libdd-data-pipeline/src/trace_exporter/mod.rs libdd-data-pipeline/tests/test_trace_exporter_otlp_protobuf_export.rs -git commit -m "feat(data-pipeline): dispatch OTLP encoder by protocol + protobuf test" -``` - -### Task 7: Builder `set_otlp_protocol` - -**Files:** -- Modify: `libdd-data-pipeline/src/trace_exporter/builder.rs` - -- [ ] **Step 1: Add the builder field + setter + use it in `build`** - -In `libdd-data-pipeline/src/trace_exporter/builder.rs`: -- add a field `otlp_protocol: OtlpProtocol` (defaults to `OtlpProtocol::default()` = `HttpJson`) to the builder struct and its `Default`/initialization; -- add the setter near `set_otlp_endpoint`: - -```rust - /// Selects the OTLP export protocol. Accepts `OtlpProtocol::HttpJson` (default) or - /// `OtlpProtocol::HttpProtobuf`. The host language resolves this from - /// `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` / `OTEL_EXPORTER_OTLP_PROTOCOL`. - pub fn set_otlp_protocol(&mut self, protocol: OtlpProtocol) -> &mut Self { - self.otlp_protocol = protocol; - self - } -``` -- in the `OtlpTraceConfig { ... }` construction, replace `protocol: OtlpProtocol::HttpJson` with `protocol: self.otlp_protocol`. - -- [ ] **Step 2: Verify it compiles** - -Run: `cargo check -p libdd-data-pipeline` -Expected: success. - -- [ ] **Step 3: Run Task 6's protobuf integration test now that the setter exists** - -Run: `cargo nextest run -p libdd-data-pipeline otlp` -Expected: PASS (both JSON and protobuf OTLP tests). - -- [ ] **Step 4: Commit** - -```bash -git add libdd-data-pipeline/src/trace_exporter/builder.rs -git commit -m "feat(data-pipeline): add TraceExporterBuilder::set_otlp_protocol" -``` - ---- - -## Phase 4 — C FFI (`libdd-data-pipeline-ffi`) - -### Task 8: `ddog_trace_exporter_config_set_otlp_protocol` - -**Files:** -- Modify: `libdd-data-pipeline-ffi/src/trace_exporter.rs` - -- [ ] **Step 1: Add the config field** - -In the `TraceExporterConfig` FFI struct (near the `otlp_endpoint: Option` field, ~line 85), add: -```rust - otlp_protocol: Option, -``` - -- [ ] **Step 2: Add the setter, modeled on `ddog_trace_exporter_config_set_otlp_endpoint`** - -After the existing OTLP endpoint setter (~line 499): - -```rust -/// Sets the OTLP export protocol. Accepts the OTel-standard values `http/json` (default) or -/// `http/protobuf`. `grpc` is rejected as not yet supported. The host language is responsible for -/// resolving the value (e.g. `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`). -#[no_mangle] -pub unsafe extern "C" fn ddog_trace_exporter_config_set_otlp_protocol( - config: Option<&mut TraceExporterConfig>, - protocol: CharSlice, -) -> Option> { - catch_panic!( - if let Some(handle) = config { - let value = match sanitize_string(protocol) { - Ok(s) => s, - Err(e) => return Some(e), - }; - match value.as_str() { - "http/json" | "http/protobuf" => { - handle.otlp_protocol = Some(value); - None - } - _ => gen_error!(ErrorCode::InvalidArgument), - } - } else { - gen_error!(ErrorCode::InvalidArgument) - }, - gen_error!(ErrorCode::Panic) - ) -} -``` - -- [ ] **Step 3: Apply the protocol in the exporter create function** - -Where the create fn calls `builder.set_otlp_endpoint(url)` (~line 566), add: -```rust - if let Some(ref proto) = config.otlp_protocol { - if let Ok(p) = proto.parse::() { - builder.set_otlp_protocol(p); - } - } -``` - -- [ ] **Step 4: Verify it compiles** - -Run: `cargo check -p libdd-data-pipeline-ffi` -Expected: success. (Confirm `OtlpProtocol` is re-exported from `libdd_data_pipeline::otlp::config`; if the ffi crate has a narrower re-export, use that path.) - -- [ ] **Step 5: Regenerate the C header** - -Run: -```bash -cargo build -p libdd-data-pipeline-ffi -``` -Then regenerate headers if the repo uses a header build step (check `builder`/`tools`); otherwise confirm the cbindgen-driven header includes `ddog_trace_exporter_config_set_otlp_protocol`. - -- [ ] **Step 6: Commit** - -```bash -git add libdd-data-pipeline-ffi/src/trace_exporter.rs -git commit -m "feat(data-pipeline-ffi): add ddog_trace_exporter_config_set_otlp_protocol" -``` - ---- - -## Phase 5 — libdatadog validation + PR - -### Task 9: Full validation gauntlet - -**Files:** none (validation). - -- [ ] **Step 1: Format** - -Run: `cargo +nightly-2026-02-08 fmt --all -- --check` -Expected: no diff. If it fails, run without `--check` and re-commit. - -- [ ] **Step 2: Clippy** - -Run: `cargo +stable clippy --workspace --all-targets --all-features -- -D warnings` -Expected: no warnings. - -- [ ] **Step 3: Tests (nextest + doc)** - -Run: -```bash -cargo nextest run --workspace --no-fail-fast -cargo nextest run --workspace --all-features --exclude builder --exclude test_spawn_from_lib -cargo test --doc -``` -Expected: all pass. (If `tracing_integration_tests::` need Docker, run `-E '!test(tracing_integration_tests::)'` and note it.) - -- [ ] **Step 4: FFI examples** - -Run: `cargo ffi-test` -Expected: C/C++ examples build + run. - -- [ ] **Step 5: License CSV (if Cargo.lock changed)** - -Run: -```bash -git diff --name-only origin/main -- Cargo.lock -``` -If `Cargo.lock` is listed: -```bash -./scripts/update_license_3rdparty.sh -cargo deny check -git add Cargo.lock LICENSE-3rdparty.csv -git commit -m "chore: update 3rd-party license CSV" -``` -Expected: `cargo deny check` clean. (Likely no Cargo.lock change since no new external crates were added.) - -- [ ] **Step 6: Apache headers on new files** - -Run: `./scripts/reformat_copyright.sh` then `git status`. -Expected: new `.rs` files carry the Apache header; commit any fixes. - -### Task 10: Open the libdatadog PR - -- [ ] **Step 1: Pre-push review (mandatory)** - -Invoke the `/pre-push-review` skill on the diff. - -- [ ] **Step 2: Push the branch** - -```bash -git push -u origin brian.marks/otlp-http-protobuf-export -``` - -- [ ] **Step 3: Create the draft PR with the repo template** - -Read `.github/pull_request_template.md`, fill all sections, and: -```bash -gh pr create --draft --label "AI Generated" --title "feat(data-pipeline): OTLP HTTP/protobuf trace export" --body-file -``` - -- [ ] **Step 4: Babysit CI** - -Invoke `/dd:pr-babysit` until CI is green (excluding `devflow/mergegate`). - ---- - -## Phase 6 — dd-trace-py wiring + local E2E (Tier 1) - -### Task 11: Set up a dd-trace-py worktree pointed at local libdatadog - -**Files:** -- Create: `/src/native/.cargo/config.toml` - -- [ ] **Step 1: Create a dd-trace-py worktree on a feature branch** - -```bash -cd /Users/brian.marks/dd/dd-trace-py -git fetch origin && git checkout main && git pull origin main -git worktree add ../dd-trace-py-otlp-protobuf -b brian.marks/otlp-http-protobuf-export -``` - -- [ ] **Step 2: Add the git-keyed cargo patch (NOT crates-io)** - -Create `../dd-trace-py-otlp-protobuf/src/native/.cargo/config.toml`: -```toml -[patch."https://github.com/DataDog/libdatadog"] -libdd-data-pipeline = { path = "/Users/brian.marks/go/src/github.com/DataDog/libdatadog-otlp-http-protobuf-export/libdd-data-pipeline" } -libdd-trace-utils = { path = "/Users/brian.marks/go/src/github.com/DataDog/libdatadog-otlp-http-protobuf-export/libdd-trace-utils" } -libdd-trace-protobuf = { path = "/Users/brian.marks/go/src/github.com/DataDog/libdatadog-otlp-http-protobuf-export/libdd-trace-protobuf" } -``` -> Add a patch line for every libdatadog crate in the modified set. If `cargo` reports an unpatched/duplicated source, add the named crate it points at. - -- [ ] **Step 3: Confirm the patch resolves** - -```bash -cd ../dd-trace-py-otlp-protobuf/src/native && cargo metadata --format-version 1 >/dev/null && echo OK -``` -Expected: `OK` (patch sources resolve). - -### Task 12: PyO3 `set_otlp_protocol` binding - -**Files:** -- Modify: `/src/native/data_pipeline/mod.rs` (after `set_otlp_headers`, ~line 189) - -- [ ] **Step 1: Add the binding, modeled on `set_otlp_endpoint`** - -```rust - fn set_otlp_protocol(mut slf: PyRefMut<'_, Self>, protocol: &'_ str) -> PyResult> { - slf.try_as_mut()?.set_otlp_protocol( - protocol - .parse() - .map_err(|e: String| pyo3::exceptions::PyValueError::new_err(e))?, - ); - Ok(slf.into()) - } -``` -> Import the builder's `OtlpProtocol` if the `.parse()` turbofish needs it: `use libdd_data_pipeline::otlp::config::OtlpProtocol;`. Match the exact `try_as_mut()` accessor used by the neighboring setters. - -- [ ] **Step 2: Build the native extension** - -```bash -cd /Users/brian.marks/dd/dd-trace-py-otlp-protobuf -python -m venv .venv && . .venv/bin/activate -pip install -e . 2>&1 | tail -20 -``` -Expected: build succeeds against the patched local libdatadog. - -- [ ] **Step 3: Smoke-test the binding from Python** - -```bash -python -c "from ddtrace.internal.native import TraceExporterBuilder as B; b=B(); b.set_otlp_protocol('http/protobuf'); print('ok')" -``` -Expected: `ok` (no exception). A bad value should raise `ValueError`. - -- [ ] **Step 4: Commit (dd-trace-py)** - -```bash -cd /Users/brian.marks/dd/dd-trace-py-otlp-protobuf -git add src/native/data_pipeline/mod.rs -git commit -m "feat(native): expose set_otlp_protocol on TraceExporterBuilder" -``` - -### Task 13: Wire `TRACES_PROTOCOL` through the writer - -**Files:** -- Modify: `/ddtrace/internal/writer/writer.py` (`_create_exporter`, ~line 827) -- Modify: `/ddtrace/internal/settings/_opentelemetry.py` (comments) - -- [ ] **Step 1: Pass the protocol when OTLP is enabled** - -In `_create_exporter`, after `builder.set_otlp_endpoint(self._otlp_endpoint)`: -```python - builder.set_otlp_protocol(otel_config.exporter.TRACES_PROTOCOL) -``` - -- [ ] **Step 2: Un-stub the comments in `_opentelemetry.py`** - -Remove the "TRACES_PROTOCOL is collected for telemetry but not yet used to switch transport" comment and update the `_derive_traces_endpoint` "libdatadog currently only supports http/json" note to reflect protobuf support. - -- [ ] **Step 3: Rebuild + commit** - -```bash -pip install -e . 2>&1 | tail -5 -git add ddtrace/internal/writer/writer.py ddtrace/internal/settings/_opentelemetry.py -git commit -m "feat(otlp): pass OTEL_EXPORTER_OTLP_TRACES_PROTOCOL to the native exporter" -``` - -### Task 14: Local protobuf-decoding receiver E2E - -**Files:** -- Create (scratch, not committed): `/tmp/otlp_recv.py`, `/tmp/otlp_app.py` - -- [ ] **Step 1: Write the receiver** - -`/tmp/otlp_recv.py`: -```python -from http.server import BaseHTTPRequestHandler, HTTPServer -from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest - -class H(BaseHTTPRequestHandler): - def do_POST(self): - n = int(self.headers.get("content-length", 0)) - body = self.rfile.read(n) - ct = self.headers.get("content-type", "") - assert ct == "application/x-protobuf", f"bad content-type: {ct}" - req = ExportTraceServiceRequest() - req.ParseFromString(body) # raises on malformed protobuf - span = req.resource_spans[0].scope_spans[0].spans[0] - print("OK decoded:", span.name, "trace_id_len", len(span.trace_id)) - self.send_response(200); self.end_headers(); self.wfile.write(b"") - -HTTPServer(("127.0.0.1", 4318), H).serve_forever() -``` -Install the proto package in the venv: `pip install opentelemetry-proto`. - -- [ ] **Step 2: Write the instrumented app** - -`/tmp/otlp_app.py`: -```python -from ddtrace import tracer -with tracer.trace("e2e_protobuf_span", resource="GET /e2e"): - pass -tracer.flush() -``` - -- [ ] **Step 3: Run protobuf E2E** - -```bash -python /tmp/otlp_recv.py & # terminal 1 -OTEL_TRACES_EXPORTER=otlp \ -OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf \ -OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://127.0.0.1:4318/v1/traces \ -python /tmp/otlp_app.py -``` -Expected: receiver prints `OK decoded: GET /e2e trace_id_len 16`. Ensure `DD_TRACE_AGENT_PROTOCOL_VERSION` is unset. - -- [ ] **Step 4: Run JSON regression E2E** - -Re-run with `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/json` and a JSON-aware receiver variant (assert `content-type: application/json`, `json.loads(body)`). -Expected: JSON path still works unchanged. - ---- - -## Phase 7 — system-tests (Tier 2) - -### Task 15: Run system-tests OTLP scenario against local builds - -**Files:** none (uses `apm-ecosystems:system-tests-local`). - -- [ ] **Step 1: Identify the OTLP trace-export scenario** - -Invoke `apm-ecosystems:system-tests-local`. In the system-tests checkout, locate the scenario(s) covering OTLP trace export / `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` for Python. Record the scenario name(s). -> This is the one item the spec left open. Resolve it here before running. - -- [ ] **Step 2: Build system-tests against the local dd-trace-py (which is built against local libdatadog)** - -Follow the skill's flow to point system-tests at the `dd-trace-py-otlp-protobuf` build. - -- [ ] **Step 3: Run the OTLP scenario with `http/protobuf`** - -Run the identified scenario; assert it passes with protocol set to `http/protobuf`. Capture output. - -- [ ] **Step 4: Record results** - -Note pass/fail and any scenario gaps in the dd-trace-py PR description. - ---- - -## Phase 8 — sdk-backend-verify (Tier 3) - -### Task 16: Full-chain backend verification - -**Files:** none (uses `apm-ecosystems:sdk-backend-verify` + the backend-integrated flow in CLAUDE.md). - -- [ ] **Step 1: Start an OTLP-capable receiver that forwards to the backend** - -Either the DD Agent with OTLP intake enabled on `:4318`, or the OTel Collector with a Datadog exporter. Use the local agent setup from CLAUDE.md (test-org API key from 1Password). Use a unique `DD_SERVICE` per run to avoid the RC/classification cache. - -- [ ] **Step 2: Emit protobuf OTLP traffic** - -```bash -OTEL_TRACES_EXPORTER=otlp \ -OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf \ -OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://127.0.0.1:4318/v1/traces \ -DD_SERVICE=bm-otlp-pb-$(date +%H%M) \ -python /tmp/otlp_app.py -``` - -- [ ] **Step 3: Verify in the backend** - -Invoke `apm-ecosystems:sdk-backend-verify` (or the spans search/aggregate APIs in CLAUDE.md) to confirm the spans landed with correct service, resource, and a 128-bit trace_id. Capture the evidence. - -- [ ] **Step 4: Record results in the dd-trace-py PR** - ---- - -## Phase 9 — dd-trace-py PR - -### Task 17: Open the dd-trace-py PR (depends on a libdatadog release) - -- [ ] **Step 1: Add the cargo dependency bump note** - -The `src/native/Cargo.toml` git pins stay at the current `rev` until libdatadog ships a release containing Phase 1–4. Document in the PR that the rev bump + removal of the local `.cargo/config.toml` patch is required before merge. Do not commit the local `.cargo/config.toml` patch. - -- [ ] **Step 2: Pre-push review + push** - -Invoke `/pre-push-review`, then push `brian.marks/otlp-http-protobuf-export`. - -- [ ] **Step 3: Create the draft PR with the repo template** - -Read dd-trace-py's PR template, fill it (including the Tier 1–3 validation evidence), and: -```bash -gh pr create --draft --label "AI Generated" --title "feat(otlp): select OTLP trace protocol (http/json|http/protobuf)" --body-file -``` - -- [ ] **Step 4: Babysit CI** - -Invoke `/dd:pr-babysit`. - ---- - -## Self-review notes (plan vs spec) - -- **Spec coverage:** type vendoring (Task 1), serde→prost converter (Task 2), encoders (Task 3), protocol `FromStr` (Task 4), content-type (Task 5), dispatch (Task 6), builder (Task 7), FFI (Task 8), validation gauntlet (Task 9), libdatadog PR (Task 10), dd-trace-py PyO3 + writer (Tasks 12–13), local E2E (Task 14), system-tests (Task 15), sdk-backend-verify (Task 16), dd-trace-py PR (Task 17). All spec sections covered. -- **Deviation:** dropped the `otlp-protobuf` cargo feature gate (justified in the header — types are unconditionally compiled via vendoring; YAGNI). -- **Known-unknown resolved in plan:** the system-tests scenario name is resolved in Task 15 Step 1 rather than left as a spec TODO. -- **Type consistency:** `OtlpProtocol` (config.rs) used consistently across Tasks 4/6/7/8/12; `encode_otlp_json`/`encode_otlp_protobuf` defined in Task 3 and used in Task 6; `ExportTraceServiceRequest` (serde) vs prost `ExportTraceServiceRequest` disambiguated via aliases. -- **Open verification points flagged inline:** exact generated prost field names (Task 2 Step 3 note), httpmock body-capture API (Task 6 Step 3 note), PyO3 `try_as_mut` accessor (Task 12 Step 1 note). diff --git a/docs/superpowers/specs/2026-06-12-otlp-http-protobuf-export-design.md b/docs/superpowers/specs/2026-06-12-otlp-http-protobuf-export-design.md deleted file mode 100644 index 8eb9c922a2..0000000000 --- a/docs/superpowers/specs/2026-06-12-otlp-http-protobuf-export-design.md +++ /dev/null @@ -1,251 +0,0 @@ -# OTLP HTTP/protobuf trace export - -- **Date:** 2026-06-12 -- **Status:** Approved design, pending implementation plan -- **Repos:** `libdatadog` (feature), `dd-trace-py` (SDK wiring + E2E) -- **Branch (libdatadog):** `brian.marks/otlp-http-protobuf-export` - -## Background - -libdatadog can export traces over OTLP, but only as **HTTP/JSON**. The trace exporter -decodes incoming (msgpack) DD spans, maps them to an OTLP `ExportTraceServiceRequest`, serializes -that to JSON, and POSTs it with `Content-Type: application/json`. - -The groundwork for more encodings already exists: - -- `OtlpProtocol::{HttpJson, HttpProtobuf, Grpc}` is stubbed in `libdd-data-pipeline/src/otlp/config.rs` - (`HttpProtobuf` and `Grpc` carry `#[allow(dead_code)]` and "not supported yet"). -- The transport (`send_otlp_traces_http`) is format-agnostic: it POSTs a `Vec` body with a - content-type header and retries. The sidecar already POSTs `application/x-protobuf` for FFE metrics. -- `libdd-common::header::APPLICATION_PROTOBUF` (`application/x-protobuf`) already exists. -- `libdd-trace-protobuf` already vendors the OTLP `common/v1` and `resource/v1` protos and generates - Rust from them via `prost-build` + `protoc-bin-vendored` behind its `generate-protobuf` feature. -- The hand-rolled serde JSON types (`libdd-trace-utils/src/otlp_encoder/json_types.rs`) deliberately - duplicate the OTLP schema; the file comment anticipates a separate protobuf path. - -dd-trace-py is already pre-wired: it reads `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` / -`OTEL_EXPORTER_OTLP_PROTOCOL` into a `TRACES_PROTOCOL` setting (validated to `http/json` / -`http/protobuf`), exposes `TraceExporterBuilder` to Python via PyO3 with `set_otlp_endpoint` / -`set_otlp_headers`, and has a comment noting `TRACES_PROTOCOL` is "collected for telemetry but not -yet used to switch transport" because libdatadog only supports JSON. - -## Goal & scope - -Add OTLP **HTTP/protobuf** as a second trace-export encoding alongside HTTP/JSON, selectable per the -OTel-standard protocol values, and wire it through dd-trace-py so it is reachable from the SDK. - -**In scope** - -- Traces only. -- Encodings: `http/json` (existing) and `http/protobuf` (new). -- Protocol selection via the Rust builder, the C FFI, and dd-trace-py's Python builder + writer. -- Validation: Rust unit/integration tests, a dd-trace-py local E2E with a protobuf-decoding receiver, - system-tests against locally-built artifacts, and sdk-backend-verify against the Datadog backend. - -**Out of scope (non-goals)** — see "Non-goals / future" for where the design leaves room: - -- gRPC transport. -- gzip / `Content-Encoding`. -- OTLP `partial_success` response parsing. -- logs / metrics signals. - -## Decisions - -1. **Type source: vendor `.proto` + generate prost types** (not the `opentelemetry-proto` crate). - Rationale: `opentelemetry-proto 0.31` aligns with the workspace's prost 0.14, but its manifest makes - `opentelemetry` and `opentelemetry_sdk` non-optional and requires `tonic` + `tonic-prost` for the - message types — it drags the OTel Rust SDK and tonic into the widely-used `libdd-trace-utils`. For a - footprint-sensitive FFI library, vendoring the protos and generating prost types via the existing - `libdd-trace-protobuf` pipeline adds **zero new runtime dependencies** and follows an established - in-repo pattern. - -2. **Keep the hand-rolled serde JSON path; do not unify onto shared types.** - Rationale: OTLP/JSON deviates from canonical protobuf-JSON (trace/span IDs are hex, not base64; - int64 is a string). The hand-rolled serde types already implement this correctly and are tested. - Generating JSON from prost types (e.g. `pbjson`) would emit base64 IDs — wrong per the OTLP/JSON - spec. So the JSON path stays exactly as-is. - -3. **Share the mapping logic via one mapper + a mechanical converter.** - Rationale: the semantic DD-span→OTLP mapping (128-bit trace-id reconstruction, span-kind inference, - attribute limits, status, flags) runs once in `map_traces_to_otlp` and produces the serde types. The - protobuf path adds only a dumb, fully-tested structural converter from the serde types to the - generated prost types. No mapping logic is duplicated. - -## Architecture & data flow - -``` -DD spans (msgpack-decoded) - │ - ▼ -map_traces_to_otlp(...) ──► ExportTraceServiceRequest (hand-rolled serde types — UNCHANGED) - │ - ├─ HttpJson ─► serde_json::to_vec(&req) ─► Content-Type: application/json - └─ HttpProtobuf ─► (&req).into() : proto::Export…Request ─► prost encode_to_vec ─► application/x-protobuf - (mechanical serde→prost converter; no mapping logic duplicated) -``` - -The endpoint path (`/v1/traces`), retry strategy, sampling enforcement (unsampled chunks dropped -before export), and resource attributes are unchanged. - -## Component changes — libdatadog - -### A. `libdd-trace-protobuf` — vendor + generate the prost types - -- Add vendored protos under `src/pb/opentelemetry/proto/`: - - `trace/v1/trace.proto` - - `collector/trace/v1/trace_service.proto` (defines `ExportTraceServiceRequest`) -- Add both to the `compile_protos([...])` list in `build.rs` (alongside the existing common/resource - entries). -- Regenerate under `--features generate-protobuf` and commit the new `opentelemetry.proto.trace.v1.rs` - and `opentelemetry.proto.collector.trace.v1.rs` (matching the checked-in-generated convention). -- Net new external runtime deps: **zero** (`prost`, `prost-build`, `protoc-bin-vendored` already present). - -### B. `libdd-trace-utils::otlp_encoder` — converter + two encoders, feature-gated - -- `json_types.rs` and `mapper.rs`: **unchanged.** -- New `proto_convert.rs`: `impl From<&ExportTraceServiceRequest> for proto::ExportTraceServiceRequest`, - converting hex-string→16/8-byte IDs, int-string→i64, base64-string→bytes, the `AnyValue` enum→prost - `any_value::Value`, dropped counts, flags, status, links, events. Behind a new `otlp-protobuf` cargo - feature that pulls the generated types from `libdd-trace-protobuf`. -- `mod.rs` exposes: - - `encode_otlp_json(&req) -> serde_json::Result>` (always available), - - `encode_otlp_protobuf(&req) -> Vec` (feature-gated). -- The feature gate keeps non-OTLP and JSON-only consumers of `libdd-trace-utils` from paying for the - protobuf types. - -### C. `libdd-data-pipeline` — protocol dispatch + config plumbing - -- `otlp/config.rs`: make `OtlpProtocol` `pub`; add `impl FromStr` (`"http/json"→HttpJson`, - `"http/protobuf"→HttpProtobuf`, `"grpc"→Grpc`); drop `#[allow(dead_code)]` on `HttpProtobuf`. -- `otlp/exporter.rs` (`send_otlp_traces_http`): set content-type from `config.protocol` - (`APPLICATION_JSON` vs `APPLICATION_PROTOBUF`) instead of hardcoding JSON; rename `json_body`→`body`. -- `trace_exporter/mod.rs` (`send_otlp_traces_inner`): replace the hardcoded `serde_json::to_vec` with a - `match config.protocol` selecting `encode_otlp_json` / `encode_otlp_protobuf`. `Grpc` returns a clear - "not yet supported" `TraceExporterError`. -- `trace_exporter/builder.rs`: add `set_otlp_protocol(OtlpProtocol)`; use it where `OtlpProtocol::HttpJson` - is currently hardcoded. Enable the `otlp-protobuf` feature on the `libdd-trace-utils` dep. - -### D. `libdd-data-pipeline-ffi` — protocol setter - -- Add `otlp_protocol` to `TraceExporterConfig` and - `ddog_trace_exporter_config_set_otlp_protocol(config, CharSlice)` that parses the OTel string via - `FromStr`, rejecting `"grpc"` with `InvalidArgument` + a clear message. -- Apply it in the create fn next to `set_otlp_endpoint`. Regenerate the C header. - -## Protocol config surface - -Mirror the OTel SDK / dd-trace-java naming: callers pass `http/json` or `http/protobuf` (the values they -read from `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` / `OTEL_EXPORTER_OTLP_PROTOCOL`). libdatadog does not read -env vars itself — the host tracer resolves the value and calls the setter, consistent with -`set_otlp_endpoint`. - -**Default = `HttpJson`**, to preserve current behavior for existing integrations. (The OTel SDK and -dd-trace-java default to `http/protobuf`; keeping JSON the default here avoids changing behavior for -callers who don't set the protocol. Easy to flip later.) - -## Component changes — dd-trace-py (companion PR) - -1. **PyO3 binding** — `src/native/data_pipeline/mod.rs`: add `set_otlp_protocol(&str)` forwarding to the - new builder method. -2. **Writer wiring** — `ddtrace/internal/writer/writer.py` `_create_exporter()`: call - `builder.set_otlp_protocol(otel_config.exporter.TRACES_PROTOCOL)` when OTLP is enabled. -3. **Un-stub the comments** — drop the "not yet used to switch transport" note at - `ddtrace/internal/settings/_opentelemetry.py` and the "libdatadog currently only supports http/json" - default note. -4. **Cargo dependency** — once libdatadog ships a release containing this feature, bump the - `rev = "v35.0.0"` git pins in `src/native/Cargo.toml`. Until then, the local cargo patch (below) is - used for E2E. - -The dd-trace-py PR is only mergeable after a libdatadog release contains the feature; it is sequenced -after the libdatadog PR. - -## Testing strategy — libdatadog - -- Existing JSON snapshot test (`otlp_export_sends_correct_payload`) and all `mapper.rs` unit tests stay - green, unchanged (JSON path untouched). -- New `proto_convert` unit tests: serde→prost equivalence (trace/span/parent IDs as bytes, kind, status, - all `AnyValue` variants incl. bytes/array, dropped counts, flags, links, events). -- New protobuf export integration test (mirrors the JSON one): mock server asserts - `Content-Type: application/x-protobuf` + path `/v1/traces`, then prost-decodes the body and asserts - `resource_spans` / `service.name` / span names. -- New parity test: `map → encode_json` vs `map → encode_protobuf → prost-decode` carry identical data — - guards the two encoders against drift. -- `FromStr` + FFI-setter tests (including `grpc` rejection). -- `cargo ffi-test` (C/C++ examples) since FFI signatures change. - -## E2E validation - -Layered, from fastest/most-deterministic to fullest-chain. - -### Tier 1 — dd-trace-py local receiver (deterministic, repeatable) - -- Point dd-trace-py at the local libdatadog build via a git-keyed cargo patch in `src/native/` - (the deps are git deps, so this is **not** `[patch.crates-io]`): - - ```toml - [patch."https://github.com/DataDog/libdatadog"] - libdd-data-pipeline = { path = "/path/to/local/libdatadog/libdd-data-pipeline" } - libdd-trace-utils = { path = "/path/to/local/libdatadog/libdd-trace-utils" } - libdd-trace-protobuf = { path = "/path/to/local/libdatadog/libdd-trace-protobuf" } - # + any other crate in the modified set - ``` - - dd-trace-py builds use libdatadog's committed generated prost code, so no `protoc` is needed there. - -- Build dd-trace-py in a fresh venv (`pip install -e .`). -- Run a small local OTLP/HTTP receiver on `:4318` handling `POST /v1/traces`: assert - `Content-Type: application/x-protobuf`, `ExportTraceServiceRequest().ParseFromString(body)` with the - `opentelemetry-proto` Python package, and assert resource `service.name`, span names, and the - 32-hex-char `trace_id` survive the round trip. -- Run a tiny instrumented app twice — `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf` and `http/json` - — confirming the new path works and the existing JSON path is unaffected. Ensure - `DD_TRACE_AGENT_PROTOCOL_VERSION` is unset (it disables OTLP). - -### Tier 2 — system-tests against local builds (via `apm-ecosystems:system-tests-local`) - -- Build dd-trace-py against the local libdatadog (Tier 1 patch), then run the relevant system-tests - OTLP scenario(s) with the locally-built tracer. The exact scenario / parametric test name is to be - identified during planning. Goal: exercise the protobuf path through the supported system-tests - harness rather than only a bespoke receiver. - -### Tier 3 — sdk-backend-verify (full chain to the Datadog backend, via `apm-ecosystems:sdk-backend-verify`) - -- Run the instrumented app with `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf` against an OTLP - receiver that forwards to the Datadog backend (DD Agent OTLP intake on `:4318`, or the OTel Collector - with a Datadog exporter), then verify the spans land in the backend with correct service/resource/ - trace-id via the backend APIs. Confirms the protobuf bytes are accepted end-to-end and ingested. - -## Validation gauntlet (per AGENTS.md) - -For each touched crate: `cargo check -p ` → -`cargo +nightly-2026-02-08 fmt --all -- --check` → -`cargo +stable clippy --workspace --all-targets --all-features -- -D warnings` → -`cargo nextest run` (workspace + all-features) → `cargo test --doc` → `cargo ffi-test`. -If `Cargo.lock` changes: `./scripts/update_license_3rdparty.sh` + `cargo deny check`. -Apache headers on new files via `./scripts/reformat_copyright.sh`. - -## Risks & mitigations - -- **Footprint spike (Phase 0 gate):** before real work, add the vendored protos, regenerate, and confirm - `cargo tree -p libdd-trace-utils --features otlp-protobuf` shows no new heavy crates. This is the whole - premise of decision 1 — go/no-go. -- **Converter correctness** (hex/base64/int-string round-trips): covered by the parity and converter - unit tests. -- **proto3 field presence:** prost uses `0`/empty for absent scalars; the converter must map - `Option`/empty consistently. Covered by unit tests; semantically harmless for OTLP receivers. -- **Cross-repo sequencing:** the dd-trace-py PR depends on a libdatadog release. E2E uses the local - cargo patch until then; the PR documents the required version bump. - -## Non-goals / future hooks - -- **gRPC:** `OtlpProtocol::Grpc` stays; rejected at the setter/exporter. A future addition is isolated to - the exporter plus a transport that doesn't fit today's HTTP/1 client. -- **gzip:** add later as a `Content-Encoding` on the existing body (`flate2` is already available). -- **`partial_success`:** neither dd-trace-go nor dd-trace-java parse it; keep status-only handling. - -## Sequencing / PR plan - -1. **libdatadog PR** (this branch): feature + unit/integration tests + regenerated protos + C header. -2. **Local E2E** (Tier 1) against the libdatadog branch via cargo patch in a dd-trace-py worktree. -3. **dd-trace-py PR**: PyO3 binding + writer wiring + comment cleanup; depends on a libdatadog release - bump. Validated with system-tests (Tier 2) and sdk-backend-verify (Tier 3) against local builds. From a8a305ffc2efd3a2d16bc2fbc94d3214c83de117 Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 12 Jun 2026 17:37:48 -0400 Subject: [PATCH 17/17] fix(data-pipeline): reject unsupported OTLP gRPC at build time Selecting OtlpProtocol::Grpc on the Rust builder previously built a working-looking exporter that then failed on every send with a mis-typed Internal(InvalidWorkerState) error. Reject it in build_async (covering sync build + wasm) with BuilderErrorKind::InvalidConfiguration (FFI: InvalidArgument), matching the C FFI set_otlp_protocol setter so both entry points fail fast and identically. The send-time arm stays as a defensive guard. Also replace the "skip as instructed" placeholder comment in the OTLP serde->prost converter with a real test exercising link()/event() conversion (link trace/span ID byte assembly, tracestate, and link/event attributes). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/trace_exporter/builder.rs | 30 ++++++ .../src/otlp_encoder/proto_convert.rs | 93 +++++++++++++++++-- 2 files changed, 116 insertions(+), 7 deletions(-) diff --git a/libdd-data-pipeline/src/trace_exporter/builder.rs b/libdd-data-pipeline/src/trace_exporter/builder.rs index 6f22c880cb..1bccb8c04c 100644 --- a/libdd-data-pipeline/src/trace_exporter/builder.rs +++ b/libdd-data-pipeline/src/trace_exporter/builder.rs @@ -290,6 +290,9 @@ impl TraceExporterBuilder { /// Selects the OTLP export protocol. Accepts `OtlpProtocol::HttpJson` (default) or /// `OtlpProtocol::HttpProtobuf`. The host language resolves this from /// `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` / `OTEL_EXPORTER_OTLP_PROTOCOL`. + /// + /// `OtlpProtocol::Grpc` is not supported; selecting it makes [`build`](Self::build) / + /// [`build_async`](Self::build_async) fail with [`BuilderErrorKind::InvalidConfiguration`]. pub fn set_otlp_protocol(&mut self, protocol: OtlpProtocol) -> &mut Self { self.otlp_protocol = protocol; self @@ -341,6 +344,18 @@ impl TraceExporterBuilder { )); } + // OTLP gRPC export is not implemented. Reject it here so a misconfigured exporter fails + // fast at build time with a clear `InvalidConfiguration` (FFI: `InvalidArgument`), matching + // the C FFI `set_otlp_protocol` setter, rather than erroring on every send. The send-time + // arm in `send_otlp_traces_inner` remains as a defensive guard. + if self.otlp_protocol == OtlpProtocol::Grpc { + return Err(TraceExporterError::Builder( + BuilderErrorKind::InvalidConfiguration( + "OTLP gRPC export is not supported".to_string(), + ), + )); + } + let shared_runtime = match self.shared_runtime { Some(rt) => rt, None => Self::new_shared_runtime()?, @@ -681,6 +696,21 @@ mod tests { )); } + #[test] + fn test_otlp_grpc_protocol_rejected_at_build() { + // gRPC is unsupported and must fail fast at build time (not on the first send), with the + // same `InvalidConfiguration` category the C FFI setter uses. + let mut builder = TraceExporterBuilder::default(); + builder.set_otlp_protocol(crate::otlp::OtlpProtocol::Grpc); + let result = builder.build::(); + assert!(matches!( + result, + Err(TraceExporterError::Builder( + BuilderErrorKind::InvalidConfiguration(_) + )) + )); + } + #[cfg_attr(miri, ignore)] #[test] fn test_build_with_v1_starts_inactive() { diff --git a/libdd-trace-utils/src/otlp_encoder/proto_convert.rs b/libdd-trace-utils/src/otlp_encoder/proto_convert.rs index daa68565a5..a21a88428a 100644 --- a/libdd-trace-utils/src/otlp_encoder/proto_convert.rs +++ b/libdd-trace-utils/src/otlp_encoder/proto_convert.rs @@ -314,11 +314,90 @@ mod tests { ); } - // Link/Event byte-size test: - // A plain v04 Span produced by the mapper does not carry links or events unless - // span.span_links / span.span_events are populated explicitly. Building a span with - // a link requires constructing a SpanLink with real trace_id/span_id values, which - // is straightforward, but the mapper only forwards links as-is — there is no - // transformation that would exercise proto_convert beyond what the ID tests above - // already cover. We therefore skip this sub-item as instructed. + #[test] + fn converts_links_and_events_to_proto() { + use crate::span::v04::{AttributeAnyValue, AttributeArrayValue, SpanEvent, SpanLink}; + use libdd_trace_protobuf::opentelemetry::proto::common::v1::any_value::Value; + use std::collections::HashMap; + + let resource_info = OtlpResourceInfo { + service: "svc".to_string(), + ..Default::default() + }; + // A link carries its own 128-bit trace ID (high<<64 | low) and 64-bit span ID, decoded by + // `link()` via a separate `hex_to_bytes` call than the top-level span IDs. + let span: Span = Span { + trace_id: 0x1_u128, + span_id: 0x2, + name: libdd_tinybytes::BytesString::from_static("op"), + resource: libdd_tinybytes::BytesString::from_static("res"), + r#type: libdd_tinybytes::BytesString::from_static("web"), + start: 1_000_000_000, + duration: 500_000, + span_links: vec![SpanLink { + trace_id: 0x1122334455667788, + trace_id_high: 0x99AABBCCDDEEFF00, + span_id: 0x0102030405060708, + attributes: HashMap::from([( + libdd_tinybytes::BytesString::from_static("link.attr"), + libdd_tinybytes::BytesString::from_static("lv"), + )]), + tracestate: libdd_tinybytes::BytesString::from_static("ts=1"), + flags: 0, + }], + span_events: vec![SpanEvent { + time_unix_nano: 1_700_000_000_000_000_000, + name: libdd_tinybytes::BytesString::from_static("ev"), + attributes: HashMap::from([( + libdd_tinybytes::BytesString::from_static("ev.attr"), + AttributeAnyValue::SingleValue(AttributeArrayValue::String( + libdd_tinybytes::BytesString::from_static("evv"), + )), + )]), + }], + ..Default::default() + }; + + let serde_req = map_traces_to_otlp(vec![vec![span]], &resource_info); + let proto: ProtoReq = (&serde_req).into(); + let sp = &proto.resource_spans[0].scope_spans[0].spans[0]; + + // --- link --- + let link = &sp.links[0]; + assert_eq!( + link.trace_id, + vec![ + 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, + 0x77, 0x88 + ] + ); + assert_eq!( + link.span_id, + vec![0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08] + ); + assert_eq!(link.trace_state, "ts=1"); + let link_attr = link + .attributes + .iter() + .find(|kv| kv.key == "link.attr") + .expect("link attr"); + assert!(matches!( + link_attr.value.as_ref().and_then(|v| v.value.as_ref()), + Some(Value::StringValue(s)) if s == "lv" + )); + + // --- event --- + let event = &sp.events[0]; + assert_eq!(event.time_unix_nano, 1_700_000_000_000_000_000); + assert_eq!(event.name, "ev"); + let event_attr = event + .attributes + .iter() + .find(|kv| kv.key == "ev.attr") + .expect("event attr"); + assert!(matches!( + event_attr.value.as_ref().and_then(|v| v.value.as_ref()), + Some(Value::StringValue(s)) if s == "evv" + )); + } }