Add conditional S3 log offload and extend retention to 100 years in gateway

.github/workflows/rust-test-lsp.yml

@@ -73,6 +73,11 @@ jobs:
       # -------------------- Java / Kotlin --------------------
       - name: Install OpenJDK 17 and jq
         run: |
+          # GH ubuntu runners ship Google Chrome's apt repo, which flakes
+          # with Hash Sum mismatch. This job does not need Chrome.
+          sudo rm -f /etc/apt/sources.list.d/google-chrome.list \
+                     /etc/apt/sources.list.d/google-chrome*.list \
+                     /etc/apt/sources.list.d/google-chrome*.sources
           sudo apt-get update
           sudo apt-get install -y openjdk-17-jdk jq unzip curl
 
@@ -115,6 +120,11 @@ jobs:
       # -------------------- Ruby --------------------
       - name: Install Ruby and dependencies
         run: |
+          # Same Chrome-repo flake as the OpenJDK step — drop it again in
+          # case a later runner image re-adds the source.
+          sudo rm -f /etc/apt/sources.list.d/google-chrome.list \
+                     /etc/apt/sources.list.d/google-chrome*.list \
+                     /etc/apt/sources.list.d/google-chrome*.sources
           sudo apt-get update
           sudo apt-get install -y ruby-full build-essential
 

gateway/README.md

@@ -41,8 +41,10 @@ gateway/
 │   ├── ratelimit/     # (stub) per-(agent|user|session) rate limits — coming soon
 │   └── auth/          # macaroon verifier adapter wiring (phases 4–6)
 └── wrapper/
-    ├── go.mod         # stdlib-only Go module (separate from plugin's bifrost dep)
-    └── main.go        # PID-1 binary: owns :8181, fronts bifrost + /_plugin/*
+    ├── go.mod           # stdlib-only Go module (separate from plugin's bifrost dep)
+    ├── main.go          # PID-1 binary: owns :8181, fronts bifrost + /_plugin/*
+    ├── authsplit.go     # bearer-concat VK.macaroon split
+    └── s3config.go      # opt-in logs_store.object_storage injection at boot
 ```
 
 ## Process layout inside the container
@@ -123,6 +125,56 @@ Bifrost detects the new `env.*` reference and re-hashes; existing
 sessions get flushed (matches `loadAuthConfig` behaviour in
 bifrost-http).
 
+## S3 log offload (opt-in)
+
+By default Bifrost keeps every LLM request/response payload in the local
+SQLite `logs.db`. To offload the heavy bodies to durable AWS S3 (keeping
+searchable metadata and signature-bound identity dims — `run-id`, `user-id`,
+`agent-name`, `org-id` — in `logs.db`), set these env vars on the container:
+
+| Env | Required | Purpose |
+| --- | -------- | ------- |
+| `BIFROST_S3_BUCKET` | yes (the on-switch) | Target bucket. Absent ⇒ local logs only, config.json byte-identical to the seed. |
+| `BIFROST_S3_REGION` | recommended | AWS region (e.g. `us-east-1`). |
+| `BIFROST_S3_ACCESS_KEY_ID` / `BIFROST_S3_SECRET_ACCESS_KEY` | no | Static credentials. Omit both to use the default AWS credential chain (instance role / IRSA). |
+| `BIFROST_S3_PREFIX` | no | S3 key prefix. Default `bifrost`. Set to `bifrost/<org-id>` so a future per-org authorization fix does not require a data migration. Bifrost stores objects at `{prefix}/logs/YYYY/MM/DD/HH/{id}.json.gz`. |
+| `BIFROST_S3_ENDPOINT` | no | Custom S3-compatible endpoint (MinIO / LocalStack / R2). |
+| `BIFROST_S3_FORCE_PATH_STYLE` | no | `1`/`true`/`yes` to use path-style URLs (required for MinIO). |
+
+The wrapper injects a `logs_store.object_storage` block into `/app/data/config.json`
+on boot. Credentials are written as Bifrost `env.BIFROST_S3_*` references, never
+as plaintext, so the 0644 config file on the volume carries no secret material.
+
+`client.log_retention_days` is **36500** (~100 years). That is the single
+retention knob — do not also set `logs_store.retention_days` (Bifrost's cleaner
+treats values `< 1` as "use the 365-day default", which would silently re-enable
+purge). Existing payloads already in `logs.db` are **not** retroactively moved;
+offload applies to new traffic only.
+
+### Bucket-side requirements (mandatory)
+
+Full LLM payloads bound to identity dims are sensitive. The target bucket MUST
+have:
+
+- **SSE-KMS** encryption
+- **S3 Block Public Access** enabled
+- a **deny-non-TLS** bucket policy
+- **S3 Object Lock (compliance mode)** or a **deny-`DeleteObject`** policy, so
+  traces cannot be deleted by the injected credential
+- the IAM user/role scoped to **least privilege**: `s3:PutObject` and
+  `s3:GetObject` on this bucket+prefix only (no `s3:DeleteObject`)
+
+Bifrost's hybrid log store does **not** delete S3 objects when the metadata
+purge runs; it expects a bucket lifecycle. Object Lock / deny-DeleteObject is
+what actually keeps signed traces around.
+
+### Known limitation (pre-existing, out of scope)
+
+`/_plugin/runs/` and `/_plugin/users/` read payloads via a single shared admin
+credential to Bifrost's `/api/logs`, with no per-org ownership check. S3 offload
+does not change that authorization surface. Structure `BIFROST_S3_PREFIX` with
+an org/realm id now so a future per-org fix does not require a data migration.
+
 ## The `/_plugin/*` namespace
 
 The wrapper routes any path under `/_plugin/` to the plugin's

gateway/data/config.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://www.getbifrost.ai/schema",
   "client": {
-    "log_retention_days": 365,
+    "log_retention_days": 36500,
     "drop_excess_requests": false,
     "enforce_auth_on_inference": true
   },

gateway/docker-compose.yml

@@ -92,6 +92,19 @@ services:
       OPENROUTER_API_KEY: ${OPENROUTER_API_KEY:-}
       GOOGLE_API_KEY: ${GOOGLE_API_KEY:-}
       XAI_API_KEY: ${XAI_API_KEY:-}
+      # Optional S3 offload of LLM request/response bodies. Unset
+      # BIFROST_S3_BUCKET ⇒ local SQLite logs only (today's behaviour).
+      # When set, the wrapper injects logs_store.object_storage into
+      # config.json with env.* credential refs — never plaintext keys.
+      # See gateway/wrapper/s3config.go for bucket-side requirements
+      # (SSE-KMS, Block Public Access, deny-non-TLS, Object Lock).
+      BIFROST_S3_BUCKET: ${BIFROST_S3_BUCKET:-}
+      BIFROST_S3_REGION: ${BIFROST_S3_REGION:-}
+      BIFROST_S3_ACCESS_KEY_ID: ${BIFROST_S3_ACCESS_KEY_ID:-}
+      BIFROST_S3_SECRET_ACCESS_KEY: ${BIFROST_S3_SECRET_ACCESS_KEY:-}
+      BIFROST_S3_PREFIX: ${BIFROST_S3_PREFIX:-}
+      BIFROST_S3_ENDPOINT: ${BIFROST_S3_ENDPOINT:-}
+      BIFROST_S3_FORCE_PATH_STYLE: ${BIFROST_S3_FORCE_PATH_STYLE:-}
     depends_on:
       redis:
         condition: service_healthy

gateway/wrapper/main.go

@@ -43,11 +43,11 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"flag"
 	"fmt"
-	"io"
 	"log"
 	"net/http"
 	"net/http/httputil"
@@ -287,9 +287,10 @@ func appDirFromArgs(args []string) string {
 	return defaultAppDir
 }
 
-// syncSeedConfig copies the image-baked config.json seed into the
-// app-dir volume, replacing any stale copy left there by a previous
-// image. Idempotent and safe to run on every boot.
+// syncSeedConfig materialises config.json into the app-dir volume
+// from the image-baked seed, optionally injecting a
+// logs_store.object_storage block when BIFROST_S3_BUCKET is set
+// (see s3config.go). Idempotent and safe to run on every boot.
 //
 // Why this exists
 // ---------------
@@ -335,89 +336,39 @@ func syncSeedConfig(logger *log.Logger, seedPath, appDir string) error {
 
 	dstPath := filepath.Join(appDir, "config.json")
 
-	// Skip the write when source and destination are byte-identical
-	// to avoid bumping mtime on every boot (which would mask "did the
-	// new image actually deploy" debugging by always showing a recent
-	// mtime).
-	if same, err := filesEqual(seedPath, dstPath); err == nil && same {
-		logger.Printf("config.json already in sync with seed at %s", dstPath)
-		return nil
-	}
-
-	// Write via temp file + rename for atomicity. A torn write here
-	// would leave Bifrost trying to parse half a JSON file on the
-	// next boot.
-	tmpPath := dstPath + ".tmp"
-	src, err := os.Open(seedPath)
+	seed, err := os.ReadFile(seedPath)
 	if err != nil {
-		return fmt.Errorf("open seed: %w", err)
+		return fmt.Errorf("read seed: %w", err)
 	}
-	defer src.Close()
 
-	dst, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
+	// Desired final config is the seed plus an optional logs_store
+	// object_storage block when BIFROST_S3_BUCKET is set (see
+	// s3config.go). Comparing *this* against the on-disk file (not
+	// the raw seed) keeps the no-op/mtime-skip intact in both the
+	// S3-on and S3-off cases.
+	desired, info, err := materializeConfig(seed, appDir)
 	if err != nil {
-		return fmt.Errorf("create temp %s: %w", tmpPath, err)
+		// Hostile/invalid S3 env must not be written into config.json.
+		// Fall back to the seed so the gateway still boots with local
+		// logs; the WARNING makes the misconfiguration visible.
+		logger.Printf("WARNING: S3 log offload config rejected: %v; continuing without object_storage", err)
+		desired = seed
+		info = s3OffloadInfo{}
 	}
-	if _, err := io.Copy(dst, src); err != nil {
-		_ = dst.Close()
-		_ = os.Remove(tmpPath)
-		return fmt.Errorf("copy seed -> temp: %w", err)
-	}
-	if err := dst.Sync(); err != nil {
-		_ = dst.Close()
-		_ = os.Remove(tmpPath)
-		return fmt.Errorf("fsync temp: %w", err)
-	}
-	if err := dst.Close(); err != nil {
-		_ = os.Remove(tmpPath)
-		return fmt.Errorf("close temp: %w", err)
+	logS3Offload(logger, info)
+
+	if existing, err := os.ReadFile(dstPath); err == nil && bytes.Equal(existing, desired) {
+		logger.Printf("config.json already in sync at %s", dstPath)
+		return nil
 	}
-	if err := os.Rename(tmpPath, dstPath); err != nil {
-		_ = os.Remove(tmpPath)
-		return fmt.Errorf("rename temp -> %s: %w", dstPath, err)
+
+	if err := writeFileAtomic(dstPath, desired, 0o644); err != nil {
+		return err
 	}
 	logger.Printf("synced config.json from seed (%d bytes) -> %s", srcInfo.Size(), dstPath)
 	return nil
 }
 
-// filesEqual compares two files by content. Returns (false, nil) if
-// either file is missing or differs in size; surfaces other I/O
-// errors. Used to no-op syncSeedConfig when the volume copy already
-// matches the seed.
-func filesEqual(a, b string) (bool, error) {
-	aInfo, err := os.Stat(a)
-	if err != nil {
-		return false, err
-	}
-	bInfo, err := os.Stat(b)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-		return false, err
-	}
-	if aInfo.Size() != bInfo.Size() {
-		return false, nil
-	}
-	aData, err := os.ReadFile(a)
-	if err != nil {
-		return false, err
-	}
-	bData, err := os.ReadFile(b)
-	if err != nil {
-		return false, err
-	}
-	if len(aData) != len(bData) {
-		return false, nil
-	}
-	for i := range aData {
-		if aData[i] != bData[i] {
-			return false, nil
-		}
-	}
-	return true, nil
-}
-
 // newProxy builds the public HTTP handler. Routes `/_plugin/*` to the
 // plugin's loopback server (if reachable) and everything else to
 // bifrost-http.