From 94d83cfedb2915581cc6430adb61cdaafdb218ac Mon Sep 17 00:00:00 2001 From: lym953 <10097700+lym953@users.noreply.github.com> Date: Thu, 15 Oct 2026 20:38:51 -0400 Subject: [PATCH 1/6] [SVLS-9883] Add DD_LOG_LEVEL_BY_TARGET to set log levels per target DD_LOG_LEVEL_BY_TARGET is a comma-separated list of `=` entries, such as `cold_start_duration=debug`. Each entry sets the log level of one tracing target, and DD_LOG_LEVEL stays the level of all other targets. The parsing is in datadog-serverless-logging, from serverless-components. The extension logs a warning for each entry that is not valid. The HTTP libraries h2, hyper, reqwest and rustls stay off, and an entry cannot turn them on. The extension sends its own logs to Datadog, so their logs for each request would add lines to each flush. One list, LOCKED_LOG_TARGETS, builds both their `off` rules and the locked list. The serverless-components dependencies point at the head of DataDog/serverless-components#183, which adds the locked targets. This commit replaces the earlier commits of this PR, rebuilt on top of the current #1412. Co-Authored-By: Claude Opus 5.5 --- bottlecap/Cargo.lock | 18 +++++++++++++---- bottlecap/Cargo.toml | 9 +++++---- bottlecap/LICENSE-3rdparty.csv | 1 + bottlecap/src/bin/bottlecap/main.rs | 30 ++++++++++++++++++++++------- bottlecap/src/logger/mod.rs | 3 +++ 5 files changed, 46 insertions(+), 15 deletions(-) diff --git a/bottlecap/Cargo.lock b/bottlecap/Cargo.lock index d2b64d865..13724f400 100644 --- a/bottlecap/Cargo.lock +++ b/bottlecap/Cargo.lock @@ -501,6 +501,7 @@ dependencies = [ "datadog-fips", "datadog-opentelemetry", "datadog-protos", + "datadog-serverless-logging", "ddsketch-agent", "dogstatsd", "figment", @@ -888,7 +889,7 @@ dependencies = [ [[package]] name = "datadog-agent-config" version = "0.1.0" -source = "git+https://github.com/DataDog/serverless-components?rev=8156757470d0843a99fc5d19809ae021cd2c7178#8156757470d0843a99fc5d19809ae021cd2c7178" +source = "git+https://github.com/DataDog/serverless-components?rev=d3ffd988d1d9f0444cbac22eafe91ac4030e0073#d3ffd988d1d9f0444cbac22eafe91ac4030e0073" dependencies = [ "datadog-opentelemetry", "dogstatsd", @@ -906,13 +907,14 @@ dependencies = [ [[package]] name = "datadog-agent-trace-sampler" version = "0.1.0" -source = "git+https://github.com/DataDog/serverless-components?rev=8156757470d0843a99fc5d19809ae021cd2c7178#8156757470d0843a99fc5d19809ae021cd2c7178" +source = "git+https://github.com/DataDog/serverless-components?rev=d3ffd988d1d9f0444cbac22eafe91ac4030e0073#d3ffd988d1d9f0444cbac22eafe91ac4030e0073" [[package]] name = "datadog-fips" version = "0.1.0" -source = "git+https://github.com/DataDog/serverless-components?rev=8156757470d0843a99fc5d19809ae021cd2c7178#8156757470d0843a99fc5d19809ae021cd2c7178" +source = "git+https://github.com/DataDog/serverless-components?rev=d3ffd988d1d9f0444cbac22eafe91ac4030e0073#d3ffd988d1d9f0444cbac22eafe91ac4030e0073" dependencies = [ + "aws-lc-rs", "reqwest", "rustls", "rustls-native-certs", @@ -971,6 +973,14 @@ dependencies = [ "tonic-build", ] +[[package]] +name = "datadog-serverless-logging" +version = "0.1.0" +source = "git+https://github.com/DataDog/serverless-components?rev=d3ffd988d1d9f0444cbac22eafe91ac4030e0073#d3ffd988d1d9f0444cbac22eafe91ac4030e0073" +dependencies = [ + "tracing-subscriber", +] + [[package]] name = "ddsketch-agent" version = "0.1.0" @@ -1090,7 +1100,7 @@ dependencies = [ [[package]] name = "dogstatsd" version = "0.1.0" -source = "git+https://github.com/DataDog/serverless-components?rev=8156757470d0843a99fc5d19809ae021cd2c7178#8156757470d0843a99fc5d19809ae021cd2c7178" +source = "git+https://github.com/DataDog/serverless-components?rev=d3ffd988d1d9f0444cbac22eafe91ac4030e0073#d3ffd988d1d9f0444cbac22eafe91ac4030e0073" dependencies = [ "datadog-protos", "ddsketch-agent", diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index 07b8f8e0b..7e0fdae72 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -95,10 +95,11 @@ libdd-trace-normalization = "3.0.1" libdd-trace-obfuscation = { version = "6.0.0", default-features = false } libdd-trace-stats = { version = "7.0.0", default-features = false } datadog-opentelemetry = { version = "0.5.2", default-features = false, features = ["_unstable_propagation"] } -dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "8156757470d0843a99fc5d19809ae021cd2c7178", default-features = false } -datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "8156757470d0843a99fc5d19809ae021cd2c7178", default-features = false } -datadog-agent-config = { git = "https://github.com/DataDog/serverless-components", rev = "8156757470d0843a99fc5d19809ae021cd2c7178", default-features = false } -datadog-agent-trace-sampler = { git = "https://github.com/DataDog/serverless-components", rev = "8156757470d0843a99fc5d19809ae021cd2c7178", default-features = false } +dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "d3ffd988d1d9f0444cbac22eafe91ac4030e0073", default-features = false } +datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "d3ffd988d1d9f0444cbac22eafe91ac4030e0073", default-features = false } +datadog-agent-config = { git = "https://github.com/DataDog/serverless-components", rev = "d3ffd988d1d9f0444cbac22eafe91ac4030e0073", default-features = false } +datadog-agent-trace-sampler = { git = "https://github.com/DataDog/serverless-components", rev = "d3ffd988d1d9f0444cbac22eafe91ac4030e0073", default-features = false } +datadog-serverless-logging = { git = "https://github.com/DataDog/serverless-components", rev = "d3ffd988d1d9f0444cbac22eafe91ac4030e0073", default-features = false } libddwaf = { version = "1.28.1", git = "https://github.com/DataDog/libddwaf-rust", rev = "d1534a158d976bd4f747bf9fcc58e0712d2d17fc", default-features = false, features = ["serde"] } [dev-dependencies] diff --git a/bottlecap/LICENSE-3rdparty.csv b/bottlecap/LICENSE-3rdparty.csv index 07ec6ff60..ca5d1db00 100644 --- a/bottlecap/LICENSE-3rdparty.csv +++ b/bottlecap/LICENSE-3rdparty.csv @@ -53,6 +53,7 @@ datadog-agent-trace-sampler,https://github.com/DataDog/serverless-components,Apa datadog-fips,https://github.com/DataDog/serverless-components,Apache-2.0,The datadog-fips Authors datadog-opentelemetry,https://github.com/DataDog/dd-trace-rs/tree/main/datadog-opentelemetry,Apache-2.0,Datadog Inc. datadog-protos,https://github.com/DataDog/saluki,Apache-2.0,The datadog-protos Authors +datadog-serverless-logging,https://github.com/DataDog/serverless-components,Apache-2.0,The datadog-serverless-logging Authors ddsketch-agent,https://github.com/DataDog/saluki,Apache-2.0,The ddsketch-agent Authors defmt,https://github.com/knurling-rs/defmt,MIT OR Apache-2.0,The Knurling-rs developers defmt-macros,https://github.com/knurling-rs/defmt,MIT OR Apache-2.0,The Knurling-rs developers diff --git a/bottlecap/src/bin/bottlecap/main.rs b/bottlecap/src/bin/bottlecap/main.rs index 3078b25b6..9ebe79d68 100644 --- a/bottlecap/src/bin/bottlecap/main.rs +++ b/bottlecap/src/bin/bottlecap/main.rs @@ -83,6 +83,7 @@ use bottlecap::{ }, }; use datadog_fips::reqwest_adapter::create_reqwest_client_builder; +use datadog_serverless_logging::{LEVEL_BY_TARGET_ENV_VAR, build_env_filter}; use decrypt::{ has_additional_endpoints_secrets, resolve_additional_endpoints_secrets, resolve_secrets, }; @@ -107,7 +108,6 @@ use tokio::time::Instant; use tokio::{sync::Mutex as TokioMutex, sync::mpsc::Sender}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, warn}; -use tracing_subscriber::EnvFilter; use ustr::Ustr; fn resolve_dsm_service( @@ -226,6 +226,11 @@ fn init_ustr() { }); } +/// HTTP libraries that log each request. The extension sends its own logs to Datadog, so their +/// logs would add lines to each flush. They stay off, and `DD_LOG_LEVEL_BY_TARGET` cannot turn +/// them on. +const LOCKED_LOG_TARGETS: &[&str] = &["h2", "hyper", "reqwest", "rustls"]; + fn enable_logging_subsystem() { let log_level = LogLevel::from_str( std::env::var("DD_LOG_LEVEL") @@ -234,13 +239,21 @@ fn enable_logging_subsystem() { ) .unwrap_or(LogLevel::Info); - let env_filter = format!( - "h2=off,hyper=off,reqwest=off,rustls=off,datadog-trace-mini-agent=off,{log_level:?}", - ); + let locked_off: Vec = LOCKED_LOG_TARGETS + .iter() + .map(|target| format!("{target}=off")) + .collect(); + let (env_filter, invalid_entries) = build_env_filter( + &format!( + "{},datadog-trace-mini-agent=off,{log_level:?}", + locked_off.join(",") + ), + &std::env::var(LEVEL_BY_TARGET_ENV_VAR).unwrap_or_default(), + LOCKED_LOG_TARGETS, + ) + .expect("could not parse log level in configuration"); let subscriber = tracing_subscriber::fmt::Subscriber::builder() - .with_env_filter( - EnvFilter::try_new(env_filter).expect("could not parse log level in configuration"), - ) + .with_env_filter(env_filter) .with_level(true) .with_thread_names(false) .with_thread_ids(false) @@ -252,6 +265,9 @@ fn enable_logging_subsystem() { .finish(); tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); + if !invalid_entries.is_empty() { + warn!("Ignoring invalid {LEVEL_BY_TARGET_ENV_VAR} entries: {invalid_entries:?}"); + } debug!("Logging subsystem enabled"); } diff --git a/bottlecap/src/logger/mod.rs b/bottlecap/src/logger/mod.rs index 177870494..f6d1c53ff 100644 --- a/bottlecap/src/logger/mod.rs +++ b/bottlecap/src/logger/mod.rs @@ -8,6 +8,9 @@ use tracing_subscriber::registry::LookupSpan; pub mod cold_start; +// `DD_LOG_LEVEL_BY_TARGET` matches a target by text prefix, so no target name may start with +// another one: an entry for `trace_flush` would also match `trace_flush_duration`. + /// Tracing target for the duration of each cold start step. pub const COLD_START_DURATION: &str = "cold_start_duration"; From 9b19382afe8c044b69d3bde92682683fda2d3afe Mon Sep 17 00:00:00 2001 From: lym953 <10097700+lym953@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:01:42 -0400 Subject: [PATCH 2/6] [SVLS-9883] fix: let DD_LOG_LEVEL_BY_TARGET turn on the HTTP library logs Drop the lock on h2, hyper, reqwest and rustls. The lock kept their lines out of the logs that the extension sends to Datadog, because those libraries log each log upload too. #1429 keeps their lines out of the upload instead, so this commit must merge after #1429. The four libraries stay off by default. serverless-components goes back to main (19270f5), which has no lock argument. Co-Authored-By: Claude Opus 5.5 --- bottlecap/Cargo.lock | 10 +++++----- bottlecap/Cargo.toml | 10 +++++----- bottlecap/src/bin/bottlecap/main.rs | 13 +------------ 3 files changed, 11 insertions(+), 22 deletions(-) diff --git a/bottlecap/Cargo.lock b/bottlecap/Cargo.lock index 13724f400..0784bc9e1 100644 --- a/bottlecap/Cargo.lock +++ b/bottlecap/Cargo.lock @@ -889,7 +889,7 @@ dependencies = [ [[package]] name = "datadog-agent-config" version = "0.1.0" -source = "git+https://github.com/DataDog/serverless-components?rev=d3ffd988d1d9f0444cbac22eafe91ac4030e0073#d3ffd988d1d9f0444cbac22eafe91ac4030e0073" +source = "git+https://github.com/DataDog/serverless-components?rev=19270f5dcbc2480824bdec25a1d30899edb3e09e#19270f5dcbc2480824bdec25a1d30899edb3e09e" dependencies = [ "datadog-opentelemetry", "dogstatsd", @@ -907,12 +907,12 @@ dependencies = [ [[package]] name = "datadog-agent-trace-sampler" version = "0.1.0" -source = "git+https://github.com/DataDog/serverless-components?rev=d3ffd988d1d9f0444cbac22eafe91ac4030e0073#d3ffd988d1d9f0444cbac22eafe91ac4030e0073" +source = "git+https://github.com/DataDog/serverless-components?rev=19270f5dcbc2480824bdec25a1d30899edb3e09e#19270f5dcbc2480824bdec25a1d30899edb3e09e" [[package]] name = "datadog-fips" version = "0.1.0" -source = "git+https://github.com/DataDog/serverless-components?rev=d3ffd988d1d9f0444cbac22eafe91ac4030e0073#d3ffd988d1d9f0444cbac22eafe91ac4030e0073" +source = "git+https://github.com/DataDog/serverless-components?rev=19270f5dcbc2480824bdec25a1d30899edb3e09e#19270f5dcbc2480824bdec25a1d30899edb3e09e" dependencies = [ "aws-lc-rs", "reqwest", @@ -976,7 +976,7 @@ dependencies = [ [[package]] name = "datadog-serverless-logging" version = "0.1.0" -source = "git+https://github.com/DataDog/serverless-components?rev=d3ffd988d1d9f0444cbac22eafe91ac4030e0073#d3ffd988d1d9f0444cbac22eafe91ac4030e0073" +source = "git+https://github.com/DataDog/serverless-components?rev=19270f5dcbc2480824bdec25a1d30899edb3e09e#19270f5dcbc2480824bdec25a1d30899edb3e09e" dependencies = [ "tracing-subscriber", ] @@ -1100,7 +1100,7 @@ dependencies = [ [[package]] name = "dogstatsd" version = "0.1.0" -source = "git+https://github.com/DataDog/serverless-components?rev=d3ffd988d1d9f0444cbac22eafe91ac4030e0073#d3ffd988d1d9f0444cbac22eafe91ac4030e0073" +source = "git+https://github.com/DataDog/serverless-components?rev=19270f5dcbc2480824bdec25a1d30899edb3e09e#19270f5dcbc2480824bdec25a1d30899edb3e09e" dependencies = [ "datadog-protos", "ddsketch-agent", diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index 7e0fdae72..68cf42869 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -95,11 +95,11 @@ libdd-trace-normalization = "3.0.1" libdd-trace-obfuscation = { version = "6.0.0", default-features = false } libdd-trace-stats = { version = "7.0.0", default-features = false } datadog-opentelemetry = { version = "0.5.2", default-features = false, features = ["_unstable_propagation"] } -dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "d3ffd988d1d9f0444cbac22eafe91ac4030e0073", default-features = false } -datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "d3ffd988d1d9f0444cbac22eafe91ac4030e0073", default-features = false } -datadog-agent-config = { git = "https://github.com/DataDog/serverless-components", rev = "d3ffd988d1d9f0444cbac22eafe91ac4030e0073", default-features = false } -datadog-agent-trace-sampler = { git = "https://github.com/DataDog/serverless-components", rev = "d3ffd988d1d9f0444cbac22eafe91ac4030e0073", default-features = false } -datadog-serverless-logging = { git = "https://github.com/DataDog/serverless-components", rev = "d3ffd988d1d9f0444cbac22eafe91ac4030e0073", default-features = false } +dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "19270f5dcbc2480824bdec25a1d30899edb3e09e", default-features = false } +datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "19270f5dcbc2480824bdec25a1d30899edb3e09e", default-features = false } +datadog-agent-config = { git = "https://github.com/DataDog/serverless-components", rev = "19270f5dcbc2480824bdec25a1d30899edb3e09e", default-features = false } +datadog-agent-trace-sampler = { git = "https://github.com/DataDog/serverless-components", rev = "19270f5dcbc2480824bdec25a1d30899edb3e09e", default-features = false } +datadog-serverless-logging = { git = "https://github.com/DataDog/serverless-components", rev = "19270f5dcbc2480824bdec25a1d30899edb3e09e", default-features = false } libddwaf = { version = "1.28.1", git = "https://github.com/DataDog/libddwaf-rust", rev = "d1534a158d976bd4f747bf9fcc58e0712d2d17fc", default-features = false, features = ["serde"] } [dev-dependencies] diff --git a/bottlecap/src/bin/bottlecap/main.rs b/bottlecap/src/bin/bottlecap/main.rs index 9ebe79d68..7d986f976 100644 --- a/bottlecap/src/bin/bottlecap/main.rs +++ b/bottlecap/src/bin/bottlecap/main.rs @@ -226,11 +226,6 @@ fn init_ustr() { }); } -/// HTTP libraries that log each request. The extension sends its own logs to Datadog, so their -/// logs would add lines to each flush. They stay off, and `DD_LOG_LEVEL_BY_TARGET` cannot turn -/// them on. -const LOCKED_LOG_TARGETS: &[&str] = &["h2", "hyper", "reqwest", "rustls"]; - fn enable_logging_subsystem() { let log_level = LogLevel::from_str( std::env::var("DD_LOG_LEVEL") @@ -239,17 +234,11 @@ fn enable_logging_subsystem() { ) .unwrap_or(LogLevel::Info); - let locked_off: Vec = LOCKED_LOG_TARGETS - .iter() - .map(|target| format!("{target}=off")) - .collect(); let (env_filter, invalid_entries) = build_env_filter( &format!( - "{},datadog-trace-mini-agent=off,{log_level:?}", - locked_off.join(",") + "h2=off,hyper=off,reqwest=off,rustls=off,datadog-trace-mini-agent=off,{log_level:?}" ), &std::env::var(LEVEL_BY_TARGET_ENV_VAR).unwrap_or_default(), - LOCKED_LOG_TARGETS, ) .expect("could not parse log level in configuration"); let subscriber = tracing_subscriber::fmt::Subscriber::builder() From f08e336190192abad75b49083f9a6e4c1419a4c7 Mon Sep 17 00:00:00 2001 From: lym953 <10097700+lym953@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:23:46 -0400 Subject: [PATCH 3/6] [SVLS-9883] docs: document DD_LOG_LEVEL_BY_TARGET in the logger README The setting is mainly for Datadog engineers and support, so it is documented next to the logger code and not on the Datadog documentation site. The README says which target names we keep stable, and how to find where an upload to Datadog stops with the HTTP library logs. Co-Authored-By: Claude Opus 5.5 --- bottlecap/src/logger/README.md | 60 ++++++++++++++++++++++++++++++++++ bottlecap/src/logger/mod.rs | 1 + 2 files changed, 61 insertions(+) create mode 100644 bottlecap/src/logger/README.md diff --git a/bottlecap/src/logger/README.md b/bottlecap/src/logger/README.md new file mode 100644 index 000000000..34055e2af --- /dev/null +++ b/bottlecap/src/logger/README.md @@ -0,0 +1,60 @@ +# Log Levels by Target + +## What it is for + +`DD_LOG_LEVEL_BY_TARGET` turns on more logs for one part of the extension, without the debug logs of all other parts. It is mainly for Datadog engineers and support, to collect logs for a support ticket. A customer can also set it. The Datadog documentation site does not describe it. + +## How to set it + +Set these environment variables on the Lambda function: + +| Environment variable | Value | +| --- | --- | +| `DD_LOG_LEVEL` | The level for all other logs, for example `info`. The default is `info`. | +| `DD_LOG_LEVEL_BY_TARGET` | A comma-separated list of `=` entries | + +A level is `off`, `error`, `warn`, `info`, `debug` or `trace`. For example, the value `cold_start_duration=debug` adds the duration of each cold start step to the `info` logs. + +An entry matches each target that starts with its text. For example, `h2=debug` matches all the `h2::…` targets. + +The extension skips an entry that is not valid, and logs a warning such as `Ignoring invalid DD_LOG_LEVEL_BY_TARGET entries: ["cold_start_duration=verbose"]`. An entry with no target, such as `debug`, is not valid. + +## Targets + +| Kind of target | Example entry | Logs that you get | Name can change? | +| --- | --- | --- | --- | +| Custom target that a log statement sets | `cold_start_duration=debug` | The duration of each cold start step | No. We keep these names. | +| HTTP library, a third-party crate | `hyper_util=debug,h2=debug` | Each TCP connect, and each HTTP/2 frame | Yes, when we update the crate | +| Module path in the extension | `bottlecap::traces=debug` | The debug logs of that module | Yes, when we move code | +| Crate from serverless-components | `dogstatsd=debug` | The debug logs of that crate | Yes, when we move code | + +Give a customer only the custom targets, and the HTTP library entries in the next section. The other names are internal, so their meaning can change in any release. + +The extension sends its own logs to Datadog, but it does not send the lines of the HTTP libraries `h2`, `hyper`, `reqwest` and `rustls`. Their lines go to CloudWatch only. These libraries log each request, including the requests that send logs, so sending their lines would add lines to each flush. + +## Find where an upload to Datadog stops + +Use this when invocations run until their timeout after the handler returns, and the extension's last line is `Attempting request`. + +| Environment variable | Value | +| --- | --- | +| `DD_LOG_LEVEL` | `debug` | +| `DD_LOG_LEVEL_BY_TARGET` | `hyper_util=debug,h2=debug` | + +Then read the last HTTP library lines before the stall in CloudWatch. These lines start with `{"target":"hyper_util…` or `{"target":"h2…`. + +| Last lines | Where the upload stopped | +| --- | --- | +| `connecting to :`, and no `connected to :` | TCP connect | +| `connected to :`, and no `binding client connection` | TLS handshake | +| `send frame=Data { … END_STREAM }`, and no `received frame=Headers` | The server did not reply | + +The `h2` lines come only from the HTTP/2 uploads. By default, the metrics and logs uploads use HTTP/2. With a proxy or `DD_HTTP_PROTOCOL=http1`, they use HTTP/1. The trace upload always uses HTTP/1, so it shows only the `hyper_util` connect lines. The `rustls` target writes no lines, because release builds of rustls do not include its logging code. + +`h2=debug` writes a line for each HTTP/2 frame, which increases the CloudWatch logs of the function. Remove both variables after you collect the logs. + +## For developers + +The crate that parses the entries and builds the log filter is `datadog-serverless-logging` in [serverless-components](https://github.com/DataDog/serverless-components). Its README lists the parsing rules. + +To add a custom target, add a constant next to `COLD_START_DURATION` in `mod.rs`. Because of the prefix match, the name of a new target must not start with the name of a target that exists. For example, an entry for `trace_flush` would also match `trace_flush_duration`. diff --git a/bottlecap/src/logger/mod.rs b/bottlecap/src/logger/mod.rs index f6d1c53ff..6419f4fdf 100644 --- a/bottlecap/src/logger/mod.rs +++ b/bottlecap/src/logger/mod.rs @@ -10,6 +10,7 @@ pub mod cold_start; // `DD_LOG_LEVEL_BY_TARGET` matches a target by text prefix, so no target name may start with // another one: an entry for `trace_flush` would also match `trace_flush_duration`. +// `README.md` in this folder documents `DD_LOG_LEVEL_BY_TARGET`. /// Tracing target for the duration of each cold start step. pub const COLD_START_DURATION: &str = "cold_start_duration"; From 44420564c91fc5b4bad4cda5e101d77ecbf6e920 Mon Sep 17 00:00:00 2001 From: lym953 <10097700+lym953@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:31:02 -0400 Subject: [PATCH 4/6] [SVLS-9883] docs: shorten the logger README Keep the setting, its format and the name stability rule. Co-Authored-By: Claude Opus 5.5 --- bottlecap/src/logger/README.md | 52 +++------------------------------- 1 file changed, 4 insertions(+), 48 deletions(-) diff --git a/bottlecap/src/logger/README.md b/bottlecap/src/logger/README.md index 34055e2af..3afed88d7 100644 --- a/bottlecap/src/logger/README.md +++ b/bottlecap/src/logger/README.md @@ -1,10 +1,6 @@ # Log Levels by Target -## What it is for - -`DD_LOG_LEVEL_BY_TARGET` turns on more logs for one part of the extension, without the debug logs of all other parts. It is mainly for Datadog engineers and support, to collect logs for a support ticket. A customer can also set it. The Datadog documentation site does not describe it. - -## How to set it +`DD_LOG_LEVEL_BY_TARGET` turns on more logs for one part of the extension, without the debug logs of all other parts. It is mainly for Datadog engineers and support, to collect logs for a support ticket. The Datadog documentation site does not describe it. Set these environment variables on the Lambda function: @@ -13,48 +9,8 @@ Set these environment variables on the Lambda function: | `DD_LOG_LEVEL` | The level for all other logs, for example `info`. The default is `info`. | | `DD_LOG_LEVEL_BY_TARGET` | A comma-separated list of `=` entries | -A level is `off`, `error`, `warn`, `info`, `debug` or `trace`. For example, the value `cold_start_duration=debug` adds the duration of each cold start step to the `info` logs. - -An entry matches each target that starts with its text. For example, `h2=debug` matches all the `h2::…` targets. - -The extension skips an entry that is not valid, and logs a warning such as `Ignoring invalid DD_LOG_LEVEL_BY_TARGET entries: ["cold_start_duration=verbose"]`. An entry with no target, such as `debug`, is not valid. - -## Targets - -| Kind of target | Example entry | Logs that you get | Name can change? | -| --- | --- | --- | --- | -| Custom target that a log statement sets | `cold_start_duration=debug` | The duration of each cold start step | No. We keep these names. | -| HTTP library, a third-party crate | `hyper_util=debug,h2=debug` | Each TCP connect, and each HTTP/2 frame | Yes, when we update the crate | -| Module path in the extension | `bottlecap::traces=debug` | The debug logs of that module | Yes, when we move code | -| Crate from serverless-components | `dogstatsd=debug` | The debug logs of that crate | Yes, when we move code | - -Give a customer only the custom targets, and the HTTP library entries in the next section. The other names are internal, so their meaning can change in any release. - -The extension sends its own logs to Datadog, but it does not send the lines of the HTTP libraries `h2`, `hyper`, `reqwest` and `rustls`. Their lines go to CloudWatch only. These libraries log each request, including the requests that send logs, so sending their lines would add lines to each flush. - -## Find where an upload to Datadog stops - -Use this when invocations run until their timeout after the handler returns, and the extension's last line is `Attempting request`. - -| Environment variable | Value | -| --- | --- | -| `DD_LOG_LEVEL` | `debug` | -| `DD_LOG_LEVEL_BY_TARGET` | `hyper_util=debug,h2=debug` | - -Then read the last HTTP library lines before the stall in CloudWatch. These lines start with `{"target":"hyper_util…` or `{"target":"h2…`. - -| Last lines | Where the upload stopped | -| --- | --- | -| `connecting to :`, and no `connected to :` | TCP connect | -| `connected to :`, and no `binding client connection` | TLS handshake | -| `send frame=Data { … END_STREAM }`, and no `received frame=Headers` | The server did not reply | - -The `h2` lines come only from the HTTP/2 uploads. By default, the metrics and logs uploads use HTTP/2. With a proxy or `DD_HTTP_PROTOCOL=http1`, they use HTTP/1. The trace upload always uses HTTP/1, so it shows only the `hyper_util` connect lines. The `rustls` target writes no lines, because release builds of rustls do not include its logging code. - -`h2=debug` writes a line for each HTTP/2 frame, which increases the CloudWatch logs of the function. Remove both variables after you collect the logs. - -## For developers +A level is `off`, `error`, `warn`, `info`, `debug` or `trace`. For example, `cold_start_duration=debug` adds the duration of each cold start step to the `info` logs. -The crate that parses the entries and builds the log filter is `datadog-serverless-logging` in [serverless-components](https://github.com/DataDog/serverless-components). Its README lists the parsing rules. +An entry matches each target that starts with its text. The extension skips an entry that is not valid, and logs a warning that names the entry. -To add a custom target, add a constant next to `COLD_START_DURATION` in `mod.rs`. Because of the prefix match, the name of a new target must not start with the name of a target that exists. For example, an entry for `trace_flush` would also match `trace_flush_duration`. +Only the custom targets, such as `cold_start_duration`, keep their names. The names of modules and crates can change in any release. From 630212aae4f44e4d41b6d999497b888c401eb30d Mon Sep 17 00:00:00 2001 From: lym953 <10097700+lym953@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:37:57 -0400 Subject: [PATCH 5/6] [SVLS-9883] docs: clarify the cold start example in the logger README Co-Authored-By: Claude Opus 5.5 --- bottlecap/src/logger/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bottlecap/src/logger/README.md b/bottlecap/src/logger/README.md index 3afed88d7..d7975fc69 100644 --- a/bottlecap/src/logger/README.md +++ b/bottlecap/src/logger/README.md @@ -9,7 +9,7 @@ Set these environment variables on the Lambda function: | `DD_LOG_LEVEL` | The level for all other logs, for example `info`. The default is `info`. | | `DD_LOG_LEVEL_BY_TARGET` | A comma-separated list of `=` entries | -A level is `off`, `error`, `warn`, `info`, `debug` or `trace`. For example, `cold_start_duration=debug` adds the duration of each cold start step to the `info` logs. +A level is `off`, `error`, `warn`, `info`, `debug` or `trace`. For example, if `DD_LOG_LEVEL` is `info` and `DD_LOG_LEVEL_BY_TARGET` is `cold_start_duration=debug`, the extension logs at `info` level as usual, and also logs the duration of each cold start step. An entry matches each target that starts with its text. The extension skips an entry that is not valid, and logs a warning that names the entry. From 62a8bf7b560155485b66530ac9f2f03a5c81e53d Mon Sep 17 00:00:00 2001 From: lym953 <10097700+lym953@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:40:01 -0400 Subject: [PATCH 6/6] [SVLS-9883] docs: reword the target name and invalid entry notes Co-Authored-By: Claude Opus 5.5 --- bottlecap/src/logger/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bottlecap/src/logger/README.md b/bottlecap/src/logger/README.md index d7975fc69..0c86b5aae 100644 --- a/bottlecap/src/logger/README.md +++ b/bottlecap/src/logger/README.md @@ -11,6 +11,6 @@ Set these environment variables on the Lambda function: A level is `off`, `error`, `warn`, `info`, `debug` or `trace`. For example, if `DD_LOG_LEVEL` is `info` and `DD_LOG_LEVEL_BY_TARGET` is `cold_start_duration=debug`, the extension logs at `info` level as usual, and also logs the duration of each cold start step. -An entry matches each target that starts with its text. The extension skips an entry that is not valid, and logs a warning that names the entry. +An entry matches each target that starts with its text. For an entry that is not valid, the extension logs a warning that names the entry, and skips it. -Only the custom targets, such as `cold_start_duration`, keep their names. The names of modules and crates can change in any release. +Use the targets that `mod.rs` defines, such as `cold_start_duration`. Other targets are module paths and crate names, which change when the code moves.