test infra: use shared datadog-mock-intake crate and add standalone mock-intake binary

.github/workflows/rs_ci.yml

@@ -84,10 +84,11 @@ jobs:
         run: cargo clippy --workspace --all-targets --features default
       - working-directory: bottlecap
         run: cargo clippy --workspace --all-targets --no-default-features --features fips
-      # No other job compiles the test-mode feature: it gates test-only
-      # constructors that are absent from default and fips builds.
+      # No other job compiles the test-mode or mock-intake features: they gate
+      # test-only constructors and the standalone mock-intake binary, which are
+      # absent from default and fips builds.
       - working-directory: bottlecap
-        run: cargo clippy --workspace --all-targets --features default,test-mode
+        run: cargo clippy --workspace --all-targets --features default,test-mode,mock-intake
 
   build-all:
     name: Build All

.gitlab/templates/pipeline.yaml.tpl

@@ -61,11 +61,12 @@ cargo clippy:
     - apt-get update && apt-get install -y --fix-missing --no-install-recommends golang-go
     - cd bottlecap
     # We need to do these separately because the fips feature is incompatible with the default feature.
-    - cargo clippy --workspace --features default
-    - cargo clippy --workspace --no-default-features --features fips
-    # No other job compiles the test-mode feature: it gates test-only
-    # constructors that are absent from default and fips builds.
-    - cargo clippy --workspace --features default,test-mode
+    - cargo clippy --workspace --all-targets --features default
+    - cargo clippy --workspace --all-targets --no-default-features --features fips
+    # No other job compiles the test-mode or mock-intake features: they gate
+    # test-only constructors and the standalone mock-intake binary, which are
+    # absent from default and fips builds.
+    - cargo clippy --workspace --all-targets --features default,test-mode,mock-intake
 
 {{ range $flavor := (ds "flavors").flavors }}
 

bottlecap/Cargo.toml

@@ -90,6 +90,10 @@ ddsketch-agent = { version = "0.1.0", default-features = false, git = "https://g
 libdd-capabilities = "3.0.0"
 libdd-common = { version = "5.2.0", default-features = false }
 libdd-trace-protobuf = "4.0.1"
+# Shared mock APM intake crate from serverless-components, linked only by the
+# standalone mock-intake binary behind the `mock-intake` feature. The APM /
+# DSM integration tests consume it as an ungated dev-dependency below.
+datadog-mock-intake = { git = "https://github.com/DataDog/serverless-components", rev = "b3e6a032f00a29a34ce61b50138677991d43c1e5", optional = true }
 libdd-trace-utils = { version = "10.1.0", default-features = false, features = ["mini_agent"] }
 libdd-trace-normalization = "3.0.1"
 libdd-trace-obfuscation = { version = "6.0.0", default-features = false }
@@ -113,16 +117,21 @@ tempfile = "3.20"
 # tls_cert_file mitigation). Already present transitively via reqwest's
 # hyper-rustls, so promoting it to a dev-dep adds nothing to the shipped binary.
 tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] }
-# fake-intake test harness: decode msgpack+gzip stats payloads on arrival
-rmp-serde = { version = "1.3.1", default-features = false }
-flate2 = { version = "1.1", default-features = false, features = ["rust_backend"] }
+# Shared mock APM intake for the APM / DSM integration tests. Ungated so the
+# tests run in every feature set.
+datadog-mock-intake = { git = "https://github.com/DataDog/serverless-components", rev = "b3e6a032f00a29a34ce61b50138677991d43c1e5" }
 
 [build-dependencies]
 # No external dependencies needed for the build script
 
 [[bin]]
 name = "bottlecap"
 
+[[bin]]
+name = "mock-intake"
+path = "src/bin/mock-intake/main.rs"
+required-features = ["mock-intake"]
+
 [profile.dev]
 debug = true   # same as debuginfo=2 and no stripping
 strip = false
@@ -187,6 +196,11 @@ fips = [
 # Not enabled in `default` or `fips`, so the items it gates do not appear
 # in production builds.
 test-mode = []
+# Builds the standalone mock-intake binary (`src/bin/mock-intake`). The
+# APM / DSM integration tests do not require this feature: they use
+# `datadog-mock-intake` as a dev-dependency and run in every test build.
+# Not enabled in `default` or `fips`.
+mock-intake = ["dep:datadog-mock-intake", "tokio/signal"]
 
 [lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(coverage,coverage_nightly)'] }

bottlecap/LICENSE-3rdparty.csv

@@ -51,6 +51,7 @@ darling_macro,https://github.com/TedDriggs/darling,MIT,Ted Driggs <ted.driggs@ou
 datadog-agent-config,https://github.com/DataDog/serverless-components,Apache-2.0,The datadog-agent-config Authors
 datadog-agent-trace-sampler,https://github.com/DataDog/serverless-components,Apache-2.0,The datadog-agent-trace-sampler Authors
 datadog-fips,https://github.com/DataDog/serverless-components,Apache-2.0,The datadog-fips Authors
+datadog-mock-intake,https://github.com/DataDog/serverless-components,Apache-2.0,The datadog-mock-intake Authors
 datadog-opentelemetry,https://github.com/DataDog/dd-trace-rs/tree/main/datadog-opentelemetry,Apache-2.0,Datadog Inc. <info@datadoghq.com>
 datadog-protos,https://github.com/DataDog/saluki,Apache-2.0,The datadog-protos Authors
 ddsketch-agent,https://github.com/DataDog/saluki,Apache-2.0,The ddsketch-agent Authors

bottlecap/README.md

@@ -3,6 +3,19 @@
 ## Development
 Use the `/scripts/build_bottlecap_layer.sh` and either publish it as a layer and test in Lambda or copy the binary into a container image and test there. Ask AJ or see the internal wiki for more.
 
+## Local APM debugging with mock-intake
+
+The `mock-intake` feature builds a standalone mock Datadog APM intake: a small HTTP server that accepts the same APM endpoints the extension flushes to, decodes the payloads, and reports them locally. It is gated behind a feature and is never part of `default` or `fips` production builds.
+
+```bash
+cd bottlecap
+cargo build --bin mock-intake --features mock-intake
+```
+
+The binary wraps the shared `datadog-mock-intake` crate from the `serverless-components` repository. The same crate backs the `apm_integration_test` and `dsm_integration_test` integration tests, which run in every test build via a dev-dependency.
+
+For configuration, endpoints, failure injection, request summaries, JSON dumps, and a smoke procedure, see [mock-intake/README.md](src/bin/mock-intake/README.md).
+
 ## Flush Strategies
 
 Bottlecap supports several flush strategies that control when and how observability data (metrics, logs, traces) is sent to Datadog. The strategy is configured via the `DD_SERVERLESS_FLUSH_STRATEGY` environment variable.

bottlecap/src/bin/mock-intake/README.md

@@ -0,0 +1,62 @@
+# mock-intake
+
+A standalone mock Datadog APM intake for local APM debugging: a small HTTP server that accepts the same APM endpoints the extension flushes to, decodes the payloads, and reports them locally.
+
+Built behind the `mock-intake` feature, which is never part of `default` or `fips` production builds:
+
+```bash
+cd bottlecap
+cargo build --bin mock-intake --features mock-intake
+```
+
+The binary wraps the shared `datadog-mock-intake` crate from the `serverless-components` repository. The same crate backs the `apm_integration_test` and `dsm_integration_test` integration tests, which run in every test build via a dev-dependency.
+
+## Configuration
+
+| Variable | Default | Meaning |
+|---|---|---|
+| `MOCK_INTAKE_PORT` | `8127` | Loopback listener port; `0` picks a free port (the bound address is printed at startup) |
+| `MOCK_INTAKE_FAIL_STATS_FIRST_N` | `0` | Return HTTP 500 for the first N stats request attempts |
+| `MOCK_INTAKE_DUMP_DIR` | unset | Write one JSON envelope per successfully decoded request into this directory |
+
+Malformed configuration values cause a clear error and a nonzero exit. Bind failures and dump-directory creation failures are also reported instead of being ignored.
+
+## Endpoints and response codes
+
+- `POST /api/v0.2/stats`: MessagePack `StatsPayload` (gzip, zstd, and identity accepted): `202 Accepted` when stored
+- `POST /api/v0.2/traces`: protobuf `AgentPayload` (gzip, zstd, and identity accepted): `202 Accepted` when stored
+- `POST /api/v0.1/pipeline_stats`: MessagePack DSM pipeline stats: `202 Accepted` when stored
+- Malformed payloads return `400 Bad Request` (unless inside the stats failure-injection window, which returns `500`)
+- Request bodies above the 2 MiB wire-body limit return `413 Payload Too Large` without being decoded
+
+## Failure injection semantics
+
+With `MOCK_INTAKE_FAIL_STATS_FIRST_N=N`, the first N stats *attempts* return 500 and their payloads are not captured by the intake. Attempts are counted per server instance, atomically, so concurrent requests cannot exceed the limit. Rejected attempts are still decoded, summarized, and dumped, so a rejected attempt and its successful retry are each visible with their own status. Trace and DSM requests do not consume the rejection budget.
+
+## Request summaries and stats grouping
+
+Each handled request produces one summary line on stderr (prefix `mock-intake:`) with the request identity, endpoint, content encoding (`identity` when absent), response status, and decoded payload count. For stats requests, hits are grouped by the full aggregation key: client service/env/version plus origin context (hostname, container, tags, git/image metadata, process tags, aggregation mode) and grouped dimensions (service, name, resource, type, DB type, HTTP status, gRPC status, synthetics, span kind, trace-root flag, HTTP method/endpoint, service source, peer tags, deprecated span-derived primary tags, and additional metric tags). Measurements (hits, errors, duration, sketches) and delivery metadata are excluded from the key, so matching dimensions combine across time buckets. Tag lists are sorted in the summary without modifying captured payloads.
+
+## JSON dumps
+
+With `MOCK_INTAKE_DUMP_DIR` set, each successfully decoded request attempt (including rejected stats attempts) writes one JSON file containing the request identity, endpoint, encoding, response status, and decoded payload. Trace payloads are serialized field by field because `AgentPayload` does not implement `Serialize` in `libdd-trace-protobuf` 4.0.1. DSM dumps contain only the fields the crate decodes. Filenames are collision-resistant and never overwrite earlier dumps.
+
+## Smoke procedure
+
+Point the test-mode trace processor at a local mock intake. The `bottlecap-test-mode` binary is added in [#1216](https://github.com/DataDog/datadog-lambda-extension/pull/1216) (not yet merged); build it from a checkout of that branch:
+
+```bash
+cd ../<test-mode-worktree>/bottlecap
+cargo build --bin bottlecap-test-mode --features test-mode
+
+cd ../<this-checkout>/bottlecap
+cargo run --bin mock-intake --features mock-intake
+
+# in another shell:
+DD_APM_DD_URL=http://127.0.0.1:8127 \
+DD_LAMBDA_EXTENSION_COMPUTE_STATS=true \
+DD_SERVICE=mock-intake-smoke DD_ENV=local DD_VERSION=smoke \
+./target/debug/bottlecap-test-mode
+```
+
+Unset `DD_SERVERLESS_FLUSH_STRATEGY` so flushing is manual, and make sure no ambient proxy intercepts localhost. Once `GET http://127.0.0.1:8126/info` succeeds, POST a MessagePack trace payload to `/v0.4/traces` (with `X-Datadog-Trace-Count`), then `POST /flush`. The `mock-intake:` summary lines should show the trace request and a stats request whose grouped hits match your spans, and the JSON dumps should contain both payloads.