diff --git a/pkg-r/R/commons.R b/pkg-r/R/commons.R index 89f21643..a8a6bce2 100644 --- a/pkg-r/R/commons.R +++ b/pkg-r/R/commons.R @@ -38,15 +38,15 @@ #' `options(commons.allow_unsafe_fallback = TRUE)`. These guardrails #' are not a security boundary. #' @param log Whether to capture conversation trajectories with OpenTelemetry -#' (default `FALSE`). When `TRUE`, commons enables GenAI message-content -#' capture in \pkg{ellmer} and tags each turn's spans with a conversation -#' id; the spans go wherever OTel is configured to export. On Posit Connect, +#' (default `FALSE`). When `TRUE`, commons tags each turn's spans with a +#' conversation id; the spans go wherever OTel is configured to export. +#' Set `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true` before R +#' starts so \pkg{ellmer} includes message content. On Posit Connect, #' traces land in Connect's observability store (browsable in its Trace #' Viewer); commons switches on the content's *Content Observability* #' setting itself when needed, though capture only starts once the content -#' restarts. Locally, commons configures \pkg{otelsdk}'s file exporter -#' automatically when no exporter is set up. Read trajectories back with -#' [trajectory_read()]. +#' restarts. Configure an exporter before starting R for local capture. Read +#' trajectories back with [trajectory_read()]. #' @param share_with An optional character vector of Connect usernames granted #' access to this content's trajectories when running on Posit Connect. #' Reading traces requires editor-level access, so named users are added as @@ -251,9 +251,6 @@ Commons <- R6::R6Class( ) private$tracing <- new_trajectory_tracing(log, share_with) - # Created after new_trajectory_tracing() so a fresh `log = TRUE` local - # exporter is already configured; otherwise otel::get_tracer() below - # would resolve and cache a no-op provider before tracing turns on. local_commons_span( "commons_agent_create", attributes = list( diff --git a/pkg-r/R/tracing.R b/pkg-r/R/tracing.R index af20d9f4..89f017ba 100644 --- a/pkg-r/R/tracing.R +++ b/pkg-r/R/tracing.R @@ -19,7 +19,6 @@ new_trajectory_tracing <- function( ) { rlang::check_bool(log, call = call) check_share_with(share_with, call = call) - repair_connect_trace_routing() if (!log) { if (!is.null(share_with)) { @@ -48,12 +47,6 @@ new_trajectory_tracing <- function( )) } - enable_content_capture() - - if (!otel::is_tracing_enabled() && !is_connect_runtime()) { - enable_local_tracing() - } - if (!otel::is_tracing_enabled()) { if (is_connect_runtime()) { enable_content_observability() @@ -63,6 +56,10 @@ new_trajectory_tracing <- function( return(FALSE) } + if (!content_capture_enabled()) { + return(FALSE) + } + if (!is.null(share_with)) { share_trajectory_access(share_with) } @@ -70,65 +67,6 @@ new_trajectory_tracing <- function( TRUE } -# Connect 2026.07 can overwrite content routing with its server attributes. -repair_connect_trace_routing <- function() { - if (!is_connect_runtime() || !is_installed("otel")) { - return(invisible(FALSE)) - } - - guid <- Sys.getenv("CONNECT_CONTENT_GUID") - job_key <- Sys.getenv("CONNECT_CONTENT_JOB_KEY") - if (!nzchar(guid) || !nzchar(job_key)) { - return(invisible(FALSE)) - } - - current <- Sys.getenv("OTEL_RESOURCE_ATTRIBUTES") - pairs <- strsplit(current, ",", fixed = TRUE)[[1]] - pairs <- pairs[nzchar(pairs)] - names <- sub("=.*$", "", pairs) - values <- sub("^[^=]*=", "", pairs) - - correctly_routed <- - any(names == "content.guid" & values == guid) && - any(names == "job.key" & values == job_key) - if (correctly_routed) { - return(invisible(FALSE)) - } - - pairs <- pairs[!names %in% c("content.guid", "job.key")] - pairs <- c( - pairs, - paste0("content.guid=", guid), - paste0("job.key=", job_key) - ) - Sys.setenv(OTEL_RESOURCE_ATTRIBUTES = paste(pairs, collapse = ",")) - repair_otelsdk_resource_attributes(guid, job_key) - reset_otel_tracer_provider() - refresh_ellmer_otel_cache() - invisible(TRUE) -} - -# otelsdk's C++ SDK statically caches its environment-derived resource when -# ellmer first initializes OTel. Rebuilding the provider does not refresh it. -repair_otelsdk_resource_attributes <- function(guid, job_key) { - if (!is_installed("otelsdk")) { - return(invisible(NULL)) - } - tryCatch( - { - the <- asNamespace("otelsdk")$the - attributes <- the$default_resource_attributes - if (is.environment(the) && is.list(attributes)) { - attributes[["content.guid"]] <- guid - attributes[["job.key"]] <- job_key - the$default_resource_attributes <- attributes - } - }, - error = function(err) NULL - ) - invisible(NULL) -} - # Start and activate a span for the calling frame's lifetime, ending when it # exits. Unlike trajectory logging, these setup spans aren't gated behind # `log = TRUE`: they cover product setup (data source and agent construction), @@ -214,92 +152,20 @@ local_conversation_turn_span <- function(envir = parent.frame()) { invisible(span) } -# HACK: ellmer snapshots its tracer and the GenAI content-capture flag once, -# in its .onLoad (`otel_cache_tracer()` in ellmer's R/otel.R). Because ellmer -# loads as a commons dependency, that snapshot is always taken before any -# commons code runs, and ellmer exports no way to refresh it. So after -# changing the OTEL_* environment, reach into ellmer and re-run its caching -# function. If ellmer's internals change, capture silently stays off for the -# session; setting the env vars before R starts (as `warn_tracing_disabled()` -# suggests) remains the manual path. -refresh_ellmer_otel_cache <- function() { - tryCatch( - utils::getFromNamespace("otel_cache_tracer", "ellmer")(), - error = function(err) NULL - ) - invisible(NULL) -} - -# ellmer captures message content only when this semconv env var is truthy -# ("true"/"1", matching ellmer's parsing). An explicit pre-set value is -# respected -- a deliberate opt-out must not be flipped process-wide -- like -# enable_local_tracing() respects an explicit OTEL_TRACES_EXPORTER. -enable_content_capture <- function() { +# ellmer reads this semconv setting when its namespace loads, so changing it +# while constructing an agent would be too late for the current process. +content_capture_enabled <- function() { current <- Sys.getenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT") - if (nzchar(current)) { - if (!tolower(current) %in% c("true", "1")) { - cli::cli_warn(c( - "{.envvar OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT} is set - to {.val {current}}, so logged trajectories will not include message - content.", - i = "Unset it or set it to {.val true} to capture full trajectories." - )) - } - return(invisible(FALSE)) - } - Sys.setenv(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = "true") - refresh_ellmer_otel_cache() - invisible(TRUE) -} - -# Configure otelsdk's file exporter for a local session that hasn't set up -# OTel itself. Only steps in when no exporter is configured at all: a user's -# explicit OTEL_TRACES_EXPORTER (even "none") is respected. -enable_local_tracing <- function() { - if (nzchar(Sys.getenv("OTEL_TRACES_EXPORTER"))) { - return(invisible(FALSE)) - } - if (!is_installed("otelsdk")) { - cli::cli_warn(c( - "Local trajectory logging requires the {.pkg otelsdk} package.", - i = "Install {.pkg otelsdk} to enable it." - )) - return(invisible(FALSE)) + if (tolower(current) %in% c("true", "1")) { + return(TRUE) } - - dir <- commons_traces_dir() - if (!dir.exists(dir)) { - dir.create(dir, recursive = TRUE) - } - Sys.setenv( - OTEL_TRACES_EXPORTER = "otlp/file", - OTEL_EXPORTER_OTLP_TRACES_FILE = file.path(dir, "trace-%N.jsonl") - ) - reset_otel_tracer_provider() - refresh_ellmer_otel_cache() - invisible(TRUE) -} - -# HACK: the otel package builds its default tracer provider from the OTEL_* -# environment on the first `otel::get_tracer()` call -- which ellmer's .onLoad -# triggers, before any commons code can run -- and caches it in the internal -# `otel:::the` environment. There is no public API to reconfigure it, so to -# honor env vars set after load, clear the cached provider and let the next -# `get_tracer()` rebuild it. Ordering matters: set the env vars first, then -# reset here, then refresh ellmer's snapshot (which calls `get_tracer()`). -# If otel's internals change, this quietly does nothing and -# `warn_tracing_disabled()` tells the user to configure `.Renviron` instead. -reset_otel_tracer_provider <- function() { - tryCatch( - { - the <- asNamespace("otel")$the - if (is.environment(the)) { - the$tracer_provider <- NULL - } - }, - error = function(err) NULL - ) - invisible(NULL) + cli::cli_warn(c( + "Trajectory logging requires GenAI message-content capture.", + i = "Set + {.code OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true} + in your {.file ~/.Renviron} and restart R." + )) + FALSE } # Tracing is off on Connect either because this content's Content diff --git a/pkg-r/inst/app.R b/pkg-r/inst/app.R index 7f7db798..5ec8d240 100644 --- a/pkg-r/inst/app.R +++ b/pkg-r/inst/app.R @@ -97,6 +97,7 @@ ui <- page_chat( server <- function(input, output, session) { agent <- commons( ellmer::chat_anthropic(), + log = TRUE, data_sources = list( warehouse = data_source( observations = observations, diff --git a/pkg-r/man/commons.Rd b/pkg-r/man/commons.Rd index 5d262e7e..75ad7d02 100644 --- a/pkg-r/man/commons.Rd +++ b/pkg-r/man/commons.Rd @@ -50,15 +50,15 @@ best-effort R guardrails with are not a security boundary.} \item{log}{Whether to capture conversation trajectories with OpenTelemetry -(default \code{FALSE}). When \code{TRUE}, commons enables GenAI message-content -capture in \pkg{ellmer} and tags each turn's spans with a conversation -id; the spans go wherever OTel is configured to export. On Posit Connect, +(default \code{FALSE}). When \code{TRUE}, commons tags each turn's spans with a +conversation id; the spans go wherever OTel is configured to export. +Set \code{OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true} before R +starts so \pkg{ellmer} includes message content. On Posit Connect, traces land in Connect's observability store (browsable in its Trace Viewer); commons switches on the content's \emph{Content Observability} setting itself when needed, though capture only starts once the content -restarts. Locally, commons configures \pkg{otelsdk}'s file exporter -automatically when no exporter is set up. Read trajectories back with -\code{\link[=trajectory_read]{trajectory_read()}}.} +restarts. Configure an exporter before starting R for local capture. Read +trajectories back with \code{\link[=trajectory_read]{trajectory_read()}}.} \item{share_with}{An optional character vector of Connect usernames granted access to this content's trajectories when running on Posit Connect. diff --git a/pkg-r/tests/testthat/_snaps/tracing.md b/pkg-r/tests/testthat/_snaps/tracing.md index ee53cf28..8570a330 100644 --- a/pkg-r/tests/testthat/_snaps/tracing.md +++ b/pkg-r/tests/testthat/_snaps/tracing.md @@ -83,12 +83,12 @@ Could not share trajectory access with `share_with`. i no api key -# an explicit content-capture setting is respected +# content capture must be configured before R starts Code - .res <- enable_content_capture() + .res <- content_capture_enabled() Condition Warning: - `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` is set to "false", so logged trajectories will not include message content. - i Unset it or set it to "true" to capture full trajectories. + Trajectory logging requires GenAI message-content capture. + i Set `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true` in your '~/.Renviron' and restart R. diff --git a/pkg-r/tests/testthat/test-commons.R b/pkg-r/tests/testthat/test-commons.R index fb68592a..b339dfe7 100644 --- a/pkg-r/tests/testthat/test-commons.R +++ b/pkg-r/tests/testthat/test-commons.R @@ -789,6 +789,9 @@ test_that("stream_async preserves structured provider content", { test_that("stream_async records provenance at span creation and completion", { skip_if_not_installed("otelsdk") + withr::local_envvar( + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = "true" + ) path <- withr::local_tempfile(fileext = ".md") writeLines("Canopy cover is always acre-weighted for reporting.", path) local_mocked_bindings(collect_appended_tags = function(...) "B") diff --git a/pkg-r/tests/testthat/test-tracing.R b/pkg-r/tests/testthat/test-tracing.R index 3685eefd..c8ad9373 100644 --- a/pkg-r/tests/testthat/test-tracing.R +++ b/pkg-r/tests/testthat/test-tracing.R @@ -24,10 +24,6 @@ test_that("log = TRUE warns when tracing stays disabled locally", { CONNECT_CONTENT_GUID = NA, COMMONS_TRACES_DIR = "/tmp/commons-traces" ) - local_mocked_bindings( - enable_content_capture = function() NULL, - enable_local_tracing = function() invisible(FALSE) - ) expect_snapshot(.res <- new_trajectory_tracing(TRUE)) expect_false(.res) @@ -37,17 +33,12 @@ test_that("log = TRUE points at Content Observability on Connect", { skip_if_not_installed("otel") withr::local_envvar(CONNECT_CONTENT_GUID = "guid") clear_observability_attempted() - state <- new.env() - state$local_enabled <- FALSE local_mocked_bindings( - enable_content_capture = function() NULL, - enable_local_tracing = function() state$local_enabled <- TRUE, connect_client = function(...) rlang::abort("no api key") ) expect_snapshot(.res <- new_trajectory_tracing(TRUE)) expect_false(.res) - expect_false(state$local_enabled) }) test_that("tracing disabled on Connect flips the observability setting on", { @@ -57,7 +48,6 @@ test_that("tracing disabled on Connect flips the observability setting on", { state <- new.env() state$patched <- 0 local_mocked_bindings( - enable_content_capture = function() NULL, connect_client = function(...) list(server = "s", api_key = "k"), connect_content = function(client, guid) list(otel_enabled = FALSE), connect_enable_otel = function(client, guid) { @@ -82,7 +72,6 @@ test_that("an already-on observability setting warns about the restart", { state <- new.env() state$patched <- 0 local_mocked_bindings( - enable_content_capture = function() NULL, connect_client = function(...) list(server = "s", api_key = "k"), connect_content = function(client, guid) list(otel_enabled = TRUE), connect_enable_otel = function(client, guid) { @@ -96,40 +85,6 @@ test_that("an already-on observability setting warns about the restart", { expect_equal(state$patched, 0) }) -test_that("enable_local_tracing configures the file exporter when unset", { - skip_if_not_installed("otelsdk") - dir <- withr::local_tempdir() - withr::local_envvar( - OTEL_TRACES_EXPORTER = NA, - OTEL_EXPORTER_OTLP_TRACES_FILE = NA, - COMMONS_TRACES_DIR = dir - ) - local_mocked_bindings( - reset_otel_tracer_provider = function() NULL, - refresh_ellmer_otel_cache = function() NULL - ) - - enable_local_tracing() - - expect_equal(Sys.getenv("OTEL_TRACES_EXPORTER"), "otlp/file") - expect_equal( - Sys.getenv("OTEL_EXPORTER_OTLP_TRACES_FILE"), - file.path(dir, "trace-%N.jsonl") - ) -}) - -test_that("enable_local_tracing respects an explicit exporter", { - withr::local_envvar(OTEL_TRACES_EXPORTER = "none") - reset <- FALSE - local_mocked_bindings( - reset_otel_tracer_provider = function() reset <<- TRUE - ) - - expect_false(enable_local_tracing()) - expect_false(reset) - expect_equal(Sys.getenv("OTEL_TRACES_EXPORTER"), "none") -}) - test_that("traces directory can come from COMMONS_TRACES_DIR", { withr::local_envvar(COMMONS_TRACES_DIR = NA) expect_equal(commons_traces_dir(), file.path(tempdir(), "commons-traces")) @@ -201,62 +156,13 @@ test_that("share_trajectory_access warns rather than errors on failure", { expect_snapshot(share_trajectory_access("jdoe")) }) -test_that("an explicit content-capture setting is respected", { +test_that("content capture must be configured before R starts", { withr::local_envvar(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = "false") - expect_snapshot(.res <- enable_content_capture()) + expect_snapshot(.res <- content_capture_enabled()) expect_false(.res) - expect_equal( - Sys.getenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"), - "false" - ) withr::local_envvar(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = "true") - expect_no_warning(.res <- enable_content_capture()) - expect_false(.res) -}) - -test_that("content capture is enabled when unset", { - withr::local_envvar(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = NA) - local_mocked_bindings(refresh_ellmer_otel_cache = function() NULL) - - expect_true(enable_content_capture()) - expect_equal( - Sys.getenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"), - "true" - ) -}) - -test_that("Connect routing repairs otelsdk's cached resource", { - skip_on_cran() - skip_if_not_installed("otelsdk") - withr::local_envvar( - POSIT_PRODUCT = "CONNECT", - CONNECT_CONTENT_GUID = "content-guid", - CONNECT_CONTENT_JOB_KEY = "job-key", - OTEL_RESOURCE_ATTRIBUTES = "k8s.namespace.name=test" - ) - the <- asNamespace("otelsdk")$the - old_attributes <- the$default_resource_attributes - withr::defer(the$default_resource_attributes <- old_attributes) - local_mocked_bindings( - reset_otel_tracer_provider = function() NULL, - refresh_ellmer_otel_cache = function() NULL - ) - - expect_true(repair_connect_trace_routing()) - expect_equal( - Sys.getenv("OTEL_RESOURCE_ATTRIBUTES"), - paste( - "k8s.namespace.name=test", - "content.guid=content-guid", - "job.key=job-key", - sep = "," - ) - ) - expect_equal( - the$default_resource_attributes[c("content.guid", "job.key")], - list("content.guid" = "content-guid", "job.key" = "job-key") - ) + expect_true(content_capture_enabled()) }) test_that("share_with grants wait for tracing to be live", { @@ -265,7 +171,6 @@ test_that("share_with grants wait for tracing to be live", { state <- new.env() state$shared <- FALSE local_mocked_bindings( - enable_content_capture = function() NULL, enable_content_observability = function() NULL, share_trajectory_access = function(share_with) state$shared <- TRUE ) @@ -276,25 +181,6 @@ test_that("share_with grants wait for tracing to be live", { expect_false(state$shared) }) -# The tracing hacks reach into ellmer and otel internals and are written to -# degrade silently if those internals change; fail loudly here instead so an -# upgrade can't quietly stop trajectory capture. -test_that("the internals the tracing hacks rely on still exist", { - skip_on_cran() - skip_if_not_installed("otel") - - expect_true(is.function(asNamespace("ellmer")[["otel_cache_tracer"]])) - - the <- asNamespace("otel")[["the"]] - expect_true(is.environment(the)) - expect_true(exists("tracer_provider", envir = the, inherits = FALSE)) - - skip_if_not_installed("otelsdk") - sdk_the <- asNamespace("otelsdk")[["the"]] - expect_true(is.environment(sdk_the)) - expect_type(sdk_the$default_resource_attributes, "list") -}) - test_that("local_commons_span is a no-op without otel", { local_mocked_bindings(is_installed = function(pkg) FALSE) expect_null(local_commons_span("commons_test_span")) diff --git a/pkg-r/vignettes/governance.Rmd b/pkg-r/vignettes/governance.Rmd index 7f4a5549..8b93dfb1 100644 --- a/pkg-r/vignettes/governance.Rmd +++ b/pkg-r/vignettes/governance.Rmd @@ -86,8 +86,8 @@ Only include facts and source code that may be shared with both the application' ## Logging trajectories -Trajectory logging is disabled by default. With `commons(log = TRUE)`, commons asks ellmer to include message content in OpenTelemetry spans. These spans contain full conversation histories, including user messages, model responses, tool requests, and tool results. Treat them as another copy of the data available to the agent. +Trajectory logging is disabled by default. Set `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true` before R starts, then use `commons(log = TRUE)` to capture message content in OpenTelemetry spans. These spans contain full conversation histories, including user messages, model responses, tool requests, and tool results. Treat them as another copy of the data available to the agent. -On Connect, both the server administrator and application author must opt in for the data to flow. The administrator must enable OpenTelemetry and allow content instrumentation; the application author sets `commons(log = TRUE)`. commons enables [Content Observability](https://docs.posit.co/connect/user/content-settings/#content-observability) for the content item when needed, although capture begins only after the content process restarts. +On Connect, both the server administrator and application author must opt in for the data to flow. The administrator must enable OpenTelemetry and allow content instrumentation; the application author sets `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true` in the content environment and uses `commons(log = TRUE)`. commons enables [Content Observability](https://docs.posit.co/connect/user/content-settings/#content-observability) for the content item when needed, although capture begins only after the content process restarts. Content owners and collaborators can read Connect trajectories with `trajectory_read()`. If a conversation is flagged or annotated in `trajectory_review()`, the reviewer writes a Markdown document containing the complete reviewer-visible conversation and its tool activity. Store these files only in an appropriately protected location, and carry out reviews in an environment trusted to handle internal user data, such as [Posit Workbench](https://posit.co/products/enterprise/workbench). Retention and access controls for the underlying OpenTelemetry data are determined by Connect or the configured exporter, not by commons.