Query parquet/csv pins in place via DuckDB views

pkg-r/DESCRIPTION

@@ -27,6 +27,7 @@ Imports:
     duckdb (>= 1.5.4.2),
     ellmer (>= 0.4.1),
     evaluate,
+    filelock,
     glue,
     highr,
     htmltools,
@@ -52,6 +53,7 @@ Suggests:
     dplyr,
     ggplot2,
     gt,
+    nanoparquet,
     odbc,
     otel (>= 0.2.0),
     otelsdk (>= 0.2.0),

pkg-r/NAMESPACE

@@ -2,6 +2,7 @@
 
 export(commons)
 export(commons_app)
+export(commons_prewarm)
 export(commons_server)
 export(commons_theme)
 export(context_layer)

pkg-r/R/chat.R

@@ -111,14 +111,7 @@ commons_server <- function(id, client, ...) {
     attributes = list("commons.server.id" = id)
   )
 
-  # Build the context index and start the background pin-cache download
-  # during post-startup idle time (while the user reads the welcome message).
-  # Errors are swallowed: the first search retries the index build and
-  # surfaces the failure to the model, and an unwarmed pin is simply
-  # downloaded at its first use.
-  later::later(function() {
-    tryCatch(client$prewarm(), error = function(err) NULL)
-  })
+  commons_prewarm(client)
 
   chat <- shinychat::chat_server(id, client = client, ...)
   # shinychat owns the conversation identity (it sets the client's
@@ -130,6 +123,99 @@ commons_server <- function(id, client, ...) {
   chat
 }
 
+#' Pre-warm a commons agent during post-startup idle time
+#'
+#' A [commons()] agent defers two kinds of setup to first use, and exposes a
+#' `prewarm()` method for each so you can move the cost off the first
+#' question:
+#'
+#' * `agent$prewarm_context()` builds the context index (the store behind
+#'   `search_context`). The index is a persistent, content-addressed file,
+#'   so the build happens once per content version: later sessions open it
+#'   in milliseconds, and it can be built offline ahead of deployment. The
+#'   cache root resolves from the `commons.context_cache` option, the
+#'   `COMMONS_CONTEXT_CACHE` or `CONNECT_CONTENT_DATA_DIR` environment
+#'   variables, an `app_cache/commons` directory beside a Shiny app, or the
+#'   per-user cache directory, in that order. This resolution ladder (and
+#'   the content-addressed, age-pruned cache design generally) follows the
+#'   file cache in the sass package, which has run in production Shiny
+#'   deployments for years. Note that `app_cache/` is
+#'   excluded from deployed bundles (rsconnect treats it as server-side
+#'   scratch), so it is shared across the sessions of one deployment but
+#'   rebuilt after a redeploy; to ship a pre-built store with the app,
+#'   point `commons.context_cache` at a directory inside the app and run
+#'   `prewarm_context()` before deploying. The cache is capped at 256 MB
+#'   with least-recently-used eviction (option
+#'   `commons.context_cache_max_size`, in bytes; a single store larger than
+#'   the cap is kept, with a warning). Set
+#'   `options(commons.context_cache = FALSE)` (or the
+#'   `COMMONS_CONTEXT_CACHE` environment variable to `false`) to disable
+#'   persistence entirely.
+#' * `agent$prewarm_sources()` starts a background process that downloads
+#'   any uncached pins into the local pins cache (see [data_source()]).
+#'   Because the pins cache is on disk, this can also run ahead of
+#'   deployment — outside the Shiny runtime entirely — and the deployed
+#'   app reads the warmed cache.
+#'
+#' `agent$prewarm()` calls both. Call `commons_prewarm()` in a Shiny server
+#' function to defer warming to post-startup idle time, so it happens while
+#' the user reads the welcome message. Outside a running Shiny app (e.g. a
+#' pre-deploy warm-up script) there is no [later::later()] event loop, so
+#' `commons_prewarm()` warms synchronously instead. Note that
+#' `commons_prewarm()` always downgrades failures to warnings (see below),
+#' even on this synchronous path — a pre-deploy script that should fail the
+#' deploy on a cold cache must call `agent$prewarm()` directly.
+#'
+#' `prewarm()` lets failures propagate, since a direct call is typically
+#' warming caches ahead of deployment and a mere warning would sail through
+#' a deploy script. `commons_prewarm()` downgrades such failures to
+#' warnings: pre-warming is a pure optimization — everything it builds is
+#' rebuilt lazily at first use — and an error escaping the [later::later()]
+#' callback would stop the app.
+#'
+#' @param client A [commons()] agent.
+#'
+#' @return `NULL`, invisibly.
+#'
+#' @examples
+#' \dontrun{
+#' server <- function(input, output, session) {
+#'   agent <- commons(
+#'     ellmer::chat_anthropic(),
+#'     data_sources = data_source(sales = sales)
+#'   )
+#'   commons_prewarm(agent)
+#'   shinychat::chat_server("chat", client = agent)
+#' }
+#' }
+#'
+#' @export
+commons_prewarm <- function(client) {
+  check_commons_client(client)
+  # An error escaping a later::later() callback stops the Shiny app, and
+  # pre-warming is a pure optimization, so downgrade failures to warnings.
+  warm <- function() {
+    tryCatch(
+      client$prewarm(),
+      error = function(err) {
+        # Assign first: the raw message can contain braces (DuckDB errors
+        # embed JSON), which cli would try to interpolate -- and an error
+        # escaping this handler would stop the app.
+        msg <- conditionMessage(err)
+        cli::cli_warn("{msg}")
+      }
+    )
+  }
+  # later::later() only fires while an event loop is running; outside Shiny
+  # (e.g. a pre-deploy warm-up script) the callback would never run.
+  if (is_shiny_app()) {
+    later::later(warm)
+  } else {
+    warm()
+  }
+  invisible(NULL)
+}
+
 check_chat_packages <- function(call = rlang::caller_env()) {
   missing <- c("htmltools", "shiny", "shinychat")[
     !vapply(

pkg-r/R/commons.R

@@ -390,18 +390,46 @@ Commons <- R6::R6Class(
     },
 
     prewarm = function() {
+      # Pre-warming is a pure optimization (everything it builds is rebuilt
+      # or downloaded lazily at first use), but a direct call is typically
+      # warming caches ahead of deployment, so failures propagate: a cold
+      # cache should fail the deploy. commons_prewarm() downgrades failures
+      # to warnings for the Shiny idle-time path, where an escaping error
+      # would stop the app.
+      self$prewarm_context()
+      self$prewarm_sources()
+      invisible(self)
+    },
+
+    prewarm_context = function() {
       layer <- private$context_layer
       layer_state <- if (is.null(layer)) NULL else context_layer_state(layer)
       if (!is.null(layer_state) && length(layer_state$docs) > 0) {
         local_commons_span(
           "commons_context_prewarm",
           attributes = list(
             "commons.context.n_docs" = length(layer_state$docs),
-            "commons.context.cache_hit" = !is.null(layer_state$store)
+            # Probe with the side-effect-free resolver so recording telemetry
+            # can't create directories or trigger the fallback warning, and
+            # tryCatch so a resolution failure can't abort prewarming.
+            "commons.context.cache_hit" =
+              !is.null(layer_state$store) ||
+              isTRUE(tryCatch(
+                context_cache_enabled() &&
+                  file.exists(context_store_path(
+                    layer_state$docs,
+                    cache_dir = context_cache_dir()
+                  )),
+                error = function(err) FALSE
+              ))
           )
         )
         context_store(layer)
       }
+      invisible(self)
+    },
+
+    prewarm_sources = function() {
       for (source in private$sources) {
         source_prewarm(source)
       }