diff --git a/.github/workflows/rust-test-lsp.yml b/.github/workflows/rust-test-lsp.yml index 3d669c2b8..3f178678a 100644 --- a/.github/workflows/rust-test-lsp.yml +++ b/.github/workflows/rust-test-lsp.yml @@ -73,6 +73,11 @@ jobs: # -------------------- Java / Kotlin -------------------- - name: Install OpenJDK 17 and jq run: | + # GH ubuntu runners ship Google Chrome's apt repo, which flakes + # with Hash Sum mismatch. This job does not need Chrome. + sudo rm -f /etc/apt/sources.list.d/google-chrome.list \ + /etc/apt/sources.list.d/google-chrome*.list \ + /etc/apt/sources.list.d/google-chrome*.sources sudo apt-get update sudo apt-get install -y openjdk-17-jdk jq unzip curl @@ -115,6 +120,11 @@ jobs: # -------------------- Ruby -------------------- - name: Install Ruby and dependencies run: | + # Same Chrome-repo flake as the OpenJDK step — drop it again in + # case a later runner image re-adds the source. + sudo rm -f /etc/apt/sources.list.d/google-chrome.list \ + /etc/apt/sources.list.d/google-chrome*.list \ + /etc/apt/sources.list.d/google-chrome*.sources sudo apt-get update sudo apt-get install -y ruby-full build-essential diff --git a/gateway/README.md b/gateway/README.md index bbd99ceee..2758d5828 100644 --- a/gateway/README.md +++ b/gateway/README.md @@ -41,8 +41,10 @@ gateway/ │ ├── ratelimit/ # (stub) per-(agent|user|session) rate limits — coming soon │ └── auth/ # macaroon verifier adapter wiring (phases 4–6) └── wrapper/ - ├── go.mod # stdlib-only Go module (separate from plugin's bifrost dep) - └── main.go # PID-1 binary: owns :8181, fronts bifrost + /_plugin/* + ├── go.mod # stdlib-only Go module (separate from plugin's bifrost dep) + ├── main.go # PID-1 binary: owns :8181, fronts bifrost + /_plugin/* + ├── authsplit.go # bearer-concat VK.macaroon split + └── s3config.go # opt-in logs_store.object_storage injection at boot ``` ## Process layout inside the container @@ -123,6 +125,56 @@ Bifrost detects the new `env.*` reference and re-hashes; existing sessions get flushed (matches `loadAuthConfig` behaviour in bifrost-http). +## S3 log offload (opt-in) + +By default Bifrost keeps every LLM request/response payload in the local +SQLite `logs.db`. To offload the heavy bodies to durable AWS S3 (keeping +searchable metadata and signature-bound identity dims — `run-id`, `user-id`, +`agent-name`, `org-id` — in `logs.db`), set these env vars on the container: + +| Env | Required | Purpose | +| --- | -------- | ------- | +| `BIFROST_S3_BUCKET` | yes (the on-switch) | Target bucket. Absent ⇒ local logs only, config.json byte-identical to the seed. | +| `BIFROST_S3_REGION` | recommended | AWS region (e.g. `us-east-1`). | +| `BIFROST_S3_ACCESS_KEY_ID` / `BIFROST_S3_SECRET_ACCESS_KEY` | no | Static credentials. Omit both to use the default AWS credential chain (instance role / IRSA). | +| `BIFROST_S3_PREFIX` | no | S3 key prefix. Default `bifrost`. Set to `bifrost/` so a future per-org authorization fix does not require a data migration. Bifrost stores objects at `{prefix}/logs/YYYY/MM/DD/HH/{id}.json.gz`. | +| `BIFROST_S3_ENDPOINT` | no | Custom S3-compatible endpoint (MinIO / LocalStack / R2). | +| `BIFROST_S3_FORCE_PATH_STYLE` | no | `1`/`true`/`yes` to use path-style URLs (required for MinIO). | + +The wrapper injects a `logs_store.object_storage` block into `/app/data/config.json` +on boot. Credentials are written as Bifrost `env.BIFROST_S3_*` references, never +as plaintext, so the 0644 config file on the volume carries no secret material. + +`client.log_retention_days` is **36500** (~100 years). That is the single +retention knob — do not also set `logs_store.retention_days` (Bifrost's cleaner +treats values `< 1` as "use the 365-day default", which would silently re-enable +purge). Existing payloads already in `logs.db` are **not** retroactively moved; +offload applies to new traffic only. + +### Bucket-side requirements (mandatory) + +Full LLM payloads bound to identity dims are sensitive. The target bucket MUST +have: + +- **SSE-KMS** encryption +- **S3 Block Public Access** enabled +- a **deny-non-TLS** bucket policy +- **S3 Object Lock (compliance mode)** or a **deny-`DeleteObject`** policy, so + traces cannot be deleted by the injected credential +- the IAM user/role scoped to **least privilege**: `s3:PutObject` and + `s3:GetObject` on this bucket+prefix only (no `s3:DeleteObject`) + +Bifrost's hybrid log store does **not** delete S3 objects when the metadata +purge runs; it expects a bucket lifecycle. Object Lock / deny-DeleteObject is +what actually keeps signed traces around. + +### Known limitation (pre-existing, out of scope) + +`/_plugin/runs/` and `/_plugin/users/` read payloads via a single shared admin +credential to Bifrost's `/api/logs`, with no per-org ownership check. S3 offload +does not change that authorization surface. Structure `BIFROST_S3_PREFIX` with +an org/realm id now so a future per-org fix does not require a data migration. + ## The `/_plugin/*` namespace The wrapper routes any path under `/_plugin/` to the plugin's diff --git a/gateway/data/config.json b/gateway/data/config.json index 34b9f0d1b..dbb305513 100644 --- a/gateway/data/config.json +++ b/gateway/data/config.json @@ -1,7 +1,7 @@ { "$schema": "https://www.getbifrost.ai/schema", "client": { - "log_retention_days": 365, + "log_retention_days": 36500, "drop_excess_requests": false, "enforce_auth_on_inference": true }, diff --git a/gateway/docker-compose.yml b/gateway/docker-compose.yml index b92dc026e..35ba858ee 100644 --- a/gateway/docker-compose.yml +++ b/gateway/docker-compose.yml @@ -92,6 +92,19 @@ services: OPENROUTER_API_KEY: ${OPENROUTER_API_KEY:-} GOOGLE_API_KEY: ${GOOGLE_API_KEY:-} XAI_API_KEY: ${XAI_API_KEY:-} + # Optional S3 offload of LLM request/response bodies. Unset + # BIFROST_S3_BUCKET ⇒ local SQLite logs only (today's behaviour). + # When set, the wrapper injects logs_store.object_storage into + # config.json with env.* credential refs — never plaintext keys. + # See gateway/wrapper/s3config.go for bucket-side requirements + # (SSE-KMS, Block Public Access, deny-non-TLS, Object Lock). + BIFROST_S3_BUCKET: ${BIFROST_S3_BUCKET:-} + BIFROST_S3_REGION: ${BIFROST_S3_REGION:-} + BIFROST_S3_ACCESS_KEY_ID: ${BIFROST_S3_ACCESS_KEY_ID:-} + BIFROST_S3_SECRET_ACCESS_KEY: ${BIFROST_S3_SECRET_ACCESS_KEY:-} + BIFROST_S3_PREFIX: ${BIFROST_S3_PREFIX:-} + BIFROST_S3_ENDPOINT: ${BIFROST_S3_ENDPOINT:-} + BIFROST_S3_FORCE_PATH_STYLE: ${BIFROST_S3_FORCE_PATH_STYLE:-} depends_on: redis: condition: service_healthy diff --git a/gateway/wrapper/main.go b/gateway/wrapper/main.go index 571ba1a8b..189f7544e 100644 --- a/gateway/wrapper/main.go +++ b/gateway/wrapper/main.go @@ -43,11 +43,11 @@ package main import ( + "bytes" "context" "errors" "flag" "fmt" - "io" "log" "net/http" "net/http/httputil" @@ -287,9 +287,10 @@ func appDirFromArgs(args []string) string { return defaultAppDir } -// syncSeedConfig copies the image-baked config.json seed into the -// app-dir volume, replacing any stale copy left there by a previous -// image. Idempotent and safe to run on every boot. +// syncSeedConfig materialises config.json into the app-dir volume +// from the image-baked seed, optionally injecting a +// logs_store.object_storage block when BIFROST_S3_BUCKET is set +// (see s3config.go). Idempotent and safe to run on every boot. // // Why this exists // --------------- @@ -335,89 +336,39 @@ func syncSeedConfig(logger *log.Logger, seedPath, appDir string) error { dstPath := filepath.Join(appDir, "config.json") - // Skip the write when source and destination are byte-identical - // to avoid bumping mtime on every boot (which would mask "did the - // new image actually deploy" debugging by always showing a recent - // mtime). - if same, err := filesEqual(seedPath, dstPath); err == nil && same { - logger.Printf("config.json already in sync with seed at %s", dstPath) - return nil - } - - // Write via temp file + rename for atomicity. A torn write here - // would leave Bifrost trying to parse half a JSON file on the - // next boot. - tmpPath := dstPath + ".tmp" - src, err := os.Open(seedPath) + seed, err := os.ReadFile(seedPath) if err != nil { - return fmt.Errorf("open seed: %w", err) + return fmt.Errorf("read seed: %w", err) } - defer src.Close() - dst, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + // Desired final config is the seed plus an optional logs_store + // object_storage block when BIFROST_S3_BUCKET is set (see + // s3config.go). Comparing *this* against the on-disk file (not + // the raw seed) keeps the no-op/mtime-skip intact in both the + // S3-on and S3-off cases. + desired, info, err := materializeConfig(seed, appDir) if err != nil { - return fmt.Errorf("create temp %s: %w", tmpPath, err) + // Hostile/invalid S3 env must not be written into config.json. + // Fall back to the seed so the gateway still boots with local + // logs; the WARNING makes the misconfiguration visible. + logger.Printf("WARNING: S3 log offload config rejected: %v; continuing without object_storage", err) + desired = seed + info = s3OffloadInfo{} } - if _, err := io.Copy(dst, src); err != nil { - _ = dst.Close() - _ = os.Remove(tmpPath) - return fmt.Errorf("copy seed -> temp: %w", err) - } - if err := dst.Sync(); err != nil { - _ = dst.Close() - _ = os.Remove(tmpPath) - return fmt.Errorf("fsync temp: %w", err) - } - if err := dst.Close(); err != nil { - _ = os.Remove(tmpPath) - return fmt.Errorf("close temp: %w", err) + logS3Offload(logger, info) + + if existing, err := os.ReadFile(dstPath); err == nil && bytes.Equal(existing, desired) { + logger.Printf("config.json already in sync at %s", dstPath) + return nil } - if err := os.Rename(tmpPath, dstPath); err != nil { - _ = os.Remove(tmpPath) - return fmt.Errorf("rename temp -> %s: %w", dstPath, err) + + if err := writeFileAtomic(dstPath, desired, 0o644); err != nil { + return err } logger.Printf("synced config.json from seed (%d bytes) -> %s", srcInfo.Size(), dstPath) return nil } -// filesEqual compares two files by content. Returns (false, nil) if -// either file is missing or differs in size; surfaces other I/O -// errors. Used to no-op syncSeedConfig when the volume copy already -// matches the seed. -func filesEqual(a, b string) (bool, error) { - aInfo, err := os.Stat(a) - if err != nil { - return false, err - } - bInfo, err := os.Stat(b) - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } - if aInfo.Size() != bInfo.Size() { - return false, nil - } - aData, err := os.ReadFile(a) - if err != nil { - return false, err - } - bData, err := os.ReadFile(b) - if err != nil { - return false, err - } - if len(aData) != len(bData) { - return false, nil - } - for i := range aData { - if aData[i] != bData[i] { - return false, nil - } - } - return true, nil -} - // newProxy builds the public HTTP handler. Routes `/_plugin/*` to the // plugin's loopback server (if reachable) and everything else to // bifrost-http. diff --git a/gateway/wrapper/s3config.go b/gateway/wrapper/s3config.go new file mode 100644 index 000000000..9763c16ba --- /dev/null +++ b/gateway/wrapper/s3config.go @@ -0,0 +1,389 @@ +// S3 log-payload offload +// ---------------------- +// When BIFROST_S3_BUCKET is set at boot, the wrapper injects a +// logs_store.object_storage block into the materialised config.json +// so Bifrost offloads LLM request/response bodies to S3 while keeping +// searchable metadata (including signature-bound identity dims) in +// the local SQLite logs.db. Absent the bucket env, the written file +// is byte-identical to the seed — today's local-only behaviour. +// +// Credentials are ALWAYS emitted as Bifrost `env.` references +// (never as the resolved secret). The wrapper never writes plaintext +// AWS keys onto the /app/data volume. +// +// Bucket-side requirements (operator, mandatory) +// ---------------------------------------------- +// Full LLM payloads bound to identity dims are sensitive. The target +// bucket MUST have: +// +// - SSE-KMS encryption +// - S3 Block Public Access enabled +// - a deny-non-TLS bucket policy +// - S3 Object Lock (compliance mode) OR a deny-DeleteObject policy +// so traces cannot be deleted by the injected credential +// - the IAM user/role scoped to least privilege: s3:PutObject and +// s3:GetObject on this bucket+prefix only (no s3:DeleteObject) +// +// Retention is governed by a single knob: client.log_retention_days +// in the seed (36500 ≈ 100 years). Do not add a competing +// logs_store.retention_days — Bifrost's cleaner treats values < 1 as +// "use the 365-day default", which would silently re-enable purge. +// HybridLogStore.DeleteLogsBatch does not delete S3 objects (it +// expects a bucket lifecycle); Object Lock / deny-DeleteObject is +// what actually keeps payloads around. +// +// Prefix layout +// ------------- +// Default prefix is "bifrost". Operators should set BIFROST_S3_PREFIX +// to include an org/realm id (e.g. "bifrost/org_acme") so a future +// per-org authorization fix does not require a data migration. +// Bifrost's object key is `{prefix}/logs/YYYY/MM/DD/HH/{id}.json.gz`. +// +// Known limitation (pre-existing, out of scope) +// --------------------------------------------- +// `/_plugin/runs/` and `/_plugin/users/` read payloads via a single +// shared admin credential to Bifrost's `/api/logs`, with no per-org +// ownership check. This feature does not change that authorization +// surface. The prefix layout above is the forward-looking seam. +// +// Compress is intentionally omitted: Bifrost's Get() only decompresses +// when the stored object has ContentEncoding=gzip, which is set only +// when compress=true was on at write time. Ship uncompressed until +// read-back rehydration is proven in production. +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +const ( + envS3Bucket = "BIFROST_S3_BUCKET" + envS3Region = "BIFROST_S3_REGION" + envS3AccessKey = "BIFROST_S3_ACCESS_KEY_ID" + envS3SecretKey = "BIFROST_S3_SECRET_ACCESS_KEY" + envS3Prefix = "BIFROST_S3_PREFIX" + envS3Endpoint = "BIFROST_S3_ENDPOINT" + + // envS3ForcePathStyle enables S3 path-style URLs (required for + // MinIO / LocalStack). Recognised truthy values: 1, true, yes. + envS3ForcePathStyle = "BIFROST_S3_FORCE_PATH_STYLE" + + // defaultS3Prefix matches Bifrost's own objectstore default and + // the upstream withobjectstorages3 example. Override via + // BIFROST_S3_PREFIX to namespace per org/realm. + defaultS3Prefix = "bifrost" + + // envRefAccessKey / envRefSecretKey are the literal strings we + // write into config.json. Bifrost's SecretVar resolver expands + // `env.NAME` at load time. Never substitute the real values here. + envRefAccessKey = "env." + envS3AccessKey + envRefSecretKey = "env." + envS3SecretKey +) + +// s3OffloadInfo is the resolved, loggable view of S3 env. Credentials +// are never stored here — only whether they were present — so it is +// safe to print. +type s3OffloadInfo struct { + Enabled bool + Bucket string + Region string + Prefix string + Endpoint string + ForcePathStyle bool + HasAccessKey bool + HasSecretKey bool +} + +// logsStoreBlock is the typed logs_store object we marshal into +// config.json. Built with encoding/json, never string concatenation, +// so operator-supplied bucket/region/prefix cannot inject sibling +// keys or corrupt the document. +type logsStoreBlock struct { + Enabled bool `json:"enabled"` + Type string `json:"type"` + Config logsStoreSQLite `json:"config"` + ObjectStorage objectStorageBlock `json:"object_storage"` +} + +type logsStoreSQLite struct { + Path string `json:"path"` +} + +type objectStorageBlock struct { + Type string `json:"type"` + Bucket string `json:"bucket"` + Region string `json:"region,omitempty"` + Prefix string `json:"prefix,omitempty"` + AccessKeyID string `json:"access_key_id,omitempty"` + SecretAccessKey string `json:"secret_access_key,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + ForcePathStyle bool `json:"force_path_style,omitempty"` +} + +// S3 bucket names: 3–63 chars, lowercase alphanumeric + dots/hyphens, +// must start and end alphanumeric. Hostile JSON fragments fail this. +var bucketNameRe = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$`) + +// AWS region ids (us-east-1, eu-central-1, us-gov-west-1, cn-north-1). +var regionNameRe = regexp.MustCompile(`^[a-z]{2}(-[a-z0-9]+)+-\d+$`) + +// S3 key prefix: starts alphanumeric, then alnum / _ . - /. No JSON +// metacharacters, no spaces, no `..` path segments (checked separately). +var prefixRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9/_.-]*$`) + +// Custom S3 endpoint (MinIO / LocalStack / R2). Scheme-optional host +// plus optional port and path. Quotes / braces / spaces rejected. +var endpointRe = regexp.MustCompile(`^(https?://)?[A-Za-z0-9._-]+(:[0-9]{1,5})?(/[A-Za-z0-9._/-]*)?$`) + +func readS3OffloadEnv() s3OffloadInfo { + info := s3OffloadInfo{ + Bucket: strings.TrimSpace(os.Getenv(envS3Bucket)), + Region: strings.TrimSpace(os.Getenv(envS3Region)), + Prefix: strings.TrimSpace(os.Getenv(envS3Prefix)), + Endpoint: strings.TrimSpace(os.Getenv(envS3Endpoint)), + HasAccessKey: strings.TrimSpace(os.Getenv(envS3AccessKey)) != "", + HasSecretKey: strings.TrimSpace(os.Getenv(envS3SecretKey)) != "", + ForcePathStyle: isTruthy(os.Getenv(envS3ForcePathStyle)), + } + if info.Prefix == "" { + info.Prefix = defaultS3Prefix + } else { + info.Prefix = strings.Trim(info.Prefix, "/") + if info.Prefix == "" { + info.Prefix = defaultS3Prefix + } + } + info.Enabled = info.Bucket != "" + return info +} + +func isTruthy(v string) bool { + switch strings.ToLower(strings.TrimSpace(v)) { + case "1", "true", "yes": + return true + default: + return false + } +} + +// materializeConfig returns the bytes that should land at +// appDir/config.json: the seed as-is when S3 is off, or the seed +// plus a marshalled logs_store.object_storage block when S3 is on. +func materializeConfig(seed []byte, appDir string) ([]byte, s3OffloadInfo, error) { + return materializeConfigFrom(seed, appDir, readS3OffloadEnv()) +} + +func materializeConfigFrom(seed []byte, appDir string, info s3OffloadInfo) ([]byte, s3OffloadInfo, error) { + if !info.Enabled { + return seed, info, nil + } + if err := validateS3Offload(info); err != nil { + return nil, info, err + } + + // Unmarshal into RawMessage values so every seed section other + // than logs_store is preserved byte-for-byte. A map[string]any + // round-trip would scramble nested key order and break the + // desired-vs-on-disk idempotency compare. + var raw map[string]json.RawMessage + if err := json.Unmarshal(seed, &raw); err != nil { + return nil, info, fmt.Errorf("parse seed config: %w", err) + } + + block, err := json.Marshal(buildLogsStore(info, appDir)) + if err != nil { + return nil, info, fmt.Errorf("marshal logs_store: %w", err) + } + raw["logs_store"] = block + + out, err := encodeTopLevel(raw) + if err != nil { + return nil, info, err + } + return out, info, nil +} + +func buildLogsStore(info s3OffloadInfo, appDir string) logsStoreBlock { + obj := objectStorageBlock{ + Type: "s3", + Bucket: info.Bucket, + Region: info.Region, + Prefix: info.Prefix, + } + // Static keys → env.NAME references, never the resolved secret. + // Both omitted → Bifrost uses the default AWS credential chain + // (instance role, IRSA, env AWS_ACCESS_KEY_ID, etc.). + // Only emit the pair when BOTH env vars are present: Bifrost + // rejects a half-configured static-credential block at boot. + if info.HasAccessKey && info.HasSecretKey { + obj.AccessKeyID = envRefAccessKey + obj.SecretAccessKey = envRefSecretKey + } + if info.Endpoint != "" { + obj.Endpoint = info.Endpoint + } + if info.ForcePathStyle { + obj.ForcePathStyle = true + } + return logsStoreBlock{ + Enabled: true, + Type: "sqlite", + Config: logsStoreSQLite{ + Path: filepath.Join(appDir, "logs.db"), + }, + ObjectStorage: obj, + } +} + +// encodeTopLevel writes a JSON object with sorted keys so the output +// is deterministic across boots (Go map iteration is randomised). +// Nested values stay as the original RawMessage bytes. +func encodeTopLevel(raw map[string]json.RawMessage) ([]byte, error) { + keys := make([]string, 0, len(raw)) + for k := range raw { + keys = append(keys, k) + } + sort.Strings(keys) + + var buf bytes.Buffer + buf.WriteString("{\n") + for i, k := range keys { + keyJSON, err := json.Marshal(k) + if err != nil { + return nil, fmt.Errorf("marshal key %q: %w", k, err) + } + val, err := indentRaw(raw[k]) + if err != nil { + return nil, fmt.Errorf("encode %s: %w", k, err) + } + buf.WriteString(" ") + buf.Write(keyJSON) + buf.WriteString(": ") + buf.Write(val) + if i < len(keys)-1 { + buf.WriteByte(',') + } + buf.WriteByte('\n') + } + buf.WriteString("}\n") + return buf.Bytes(), nil +} + +// indentRaw pretty-prints a RawMessage with 2-space indent so a +// compact logs_store blob (from json.Marshal) sits at the same +// indentation as the rest of the seed. +func indentRaw(raw json.RawMessage) ([]byte, error) { + var buf bytes.Buffer + if err := json.Indent(&buf, raw, " ", " "); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func validateS3Offload(info s3OffloadInfo) error { + if err := validateBucket(info.Bucket); err != nil { + return err + } + if info.Region != "" { + if err := validateRegion(info.Region); err != nil { + return err + } + } + if err := validatePrefix(info.Prefix); err != nil { + return err + } + if info.Endpoint != "" { + if err := validateEndpoint(info.Endpoint); err != nil { + return err + } + } + return nil +} + +func validateBucket(v string) error { + if !bucketNameRe.MatchString(v) || strings.Contains(v, "..") { + return fmt.Errorf("invalid %s %q: must be a 3-63 char S3 bucket name (lowercase letters, digits, dots, hyphens)", envS3Bucket, v) + } + return nil +} + +func validateRegion(v string) error { + if !regionNameRe.MatchString(v) { + return fmt.Errorf("invalid %s %q: must be an AWS region id (e.g. us-east-1)", envS3Region, v) + } + return nil +} + +func validatePrefix(v string) error { + if !prefixRe.MatchString(v) || strings.Contains(v, "..") { + return fmt.Errorf("invalid %s %q: must be an S3 key prefix (alphanumeric, '/', '_', '.', '-')", envS3Prefix, v) + } + return nil +} + +func validateEndpoint(v string) error { + if !endpointRe.MatchString(v) { + return fmt.Errorf("invalid %s %q: must be a host[:port] or http(s) URL", envS3Endpoint, v) + } + return nil +} + +func logS3Offload(logger *log.Logger, info s3OffloadInfo) { + if !info.Enabled { + logger.Printf("S3 log offload disabled") + return + } + logger.Printf("S3 log offload enabled bucket=%s region=%s prefix=%s", + info.Bucket, info.Region, info.Prefix) + if info.Region == "" { + logger.Printf("WARNING: %s is set but %s is empty; Bifrost will use the AWS SDK default region chain", + envS3Bucket, envS3Region) + } + switch { + case info.HasAccessKey && info.HasSecretKey: + // Static keys present as env refs — nothing to warn about. + case !info.HasAccessKey && !info.HasSecretKey: + logger.Printf("WARNING: %s is set but %s/%s are empty; Bifrost will use the default AWS credential chain (instance role)", + envS3Bucket, envS3AccessKey, envS3SecretKey) + default: + logger.Printf("WARNING: %s is set but only one of %s/%s is present; both are required for static credentials", + envS3Bucket, envS3AccessKey, envS3SecretKey) + } +} + +// writeFileAtomic writes data to path via a sibling temp file + rename +// so a torn write cannot leave Bifrost parsing half a JSON document. +func writeFileAtomic(path string, data []byte, mode os.FileMode) error { + tmpPath := path + ".tmp" + dst, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) + if err != nil { + return fmt.Errorf("create temp %s: %w", tmpPath, err) + } + if _, err := dst.Write(data); err != nil { + _ = dst.Close() + _ = os.Remove(tmpPath) + return fmt.Errorf("write temp: %w", err) + } + if err := dst.Sync(); err != nil { + _ = dst.Close() + _ = os.Remove(tmpPath) + return fmt.Errorf("fsync temp: %w", err) + } + if err := dst.Close(); err != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("close temp: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("rename temp -> %s: %w", path, err) + } + return nil +} diff --git a/gateway/wrapper/s3config_integration_test.go b/gateway/wrapper/s3config_integration_test.go new file mode 100644 index 000000000..7b2f5cf9b --- /dev/null +++ b/gateway/wrapper/s3config_integration_test.go @@ -0,0 +1,68 @@ +// Integration smoke for S3 payload offload against LocalStack/MinIO. +// +// Not run in ordinary `go test ./...` — it needs a live gateway image +// booted with BIFROST_S3_* and an S3-compatible bucket. Enable with: +// +// BIFROST_S3_INTEGRATION=1 \ +// BIFROST_S3_ENDPOINT=http://localhost:4566 \ +// BIFROST_S3_BUCKET=... BIFROST_S3_REGION=us-east-1 \ +// BIFROST_S3_ACCESS_KEY_ID=test BIFROST_S3_SECRET_ACCESS_KEY=test \ +// go test -count=1 -run TestIntegrationS3Offload ./ +// +// What this proves (the T1 acceptance path that unit tests cannot): +// - wrapper materialises logs_store.object_storage with env.* refs +// - bifrost-http honors that block (no license gate) and pings S3 +// - a subsequent LLM call lands an object under {prefix}/logs/... +// - GET /api/logs rehydrates the body; identity dims stay on the +// local metadata row +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +func TestIntegrationS3Offload_MaterializeAgainstLiveEnv(t *testing.T) { + if os.Getenv("BIFROST_S3_INTEGRATION") == "" { + t.Skip("set BIFROST_S3_INTEGRATION=1 to run against LocalStack/MinIO") + } + + seedPath := filepath.Join("..", "data", "config.json") + seed, err := os.ReadFile(seedPath) + if err != nil { + t.Fatalf("read seed: %v", err) + } + + out, info, err := materializeConfig(seed, "/app/data") + if err != nil { + t.Fatalf("materialize with live env: %v", err) + } + if !info.Enabled { + t.Fatal("BIFROST_S3_INTEGRATION=1 requires BIFROST_S3_BUCKET to be set") + } + + var parsed map[string]any + if err := json.Unmarshal(out, &parsed); err != nil { + t.Fatalf("materialised config is not JSON: %v", err) + } + ls, _ := parsed["logs_store"].(map[string]any) + if ls == nil { + t.Fatal("expected logs_store in materialised config") + } + obj, _ := ls["object_storage"].(map[string]any) + if obj == nil || obj["type"] != "s3" { + t.Fatalf("expected object_storage.type=s3, got %#v", obj) + } + if obj["bucket"] != info.Bucket { + t.Fatalf("bucket = %v, want %s", obj["bucket"], info.Bucket) + } + if ak, _ := obj["access_key_id"].(string); ak != "" && ak != envRefAccessKey { + t.Fatalf("access_key_id must be an env.* ref, got %q", ak) + } + + t.Logf("materialised logs_store.object_storage bucket=%s region=%s prefix=%s endpoint=%s", + info.Bucket, info.Region, info.Prefix, info.Endpoint) + t.Log("full boot+LLM-call+object-lands-in-bucket assertion requires the gateway image; this test only checks the config the wrapper would write") +} diff --git a/gateway/wrapper/s3config_test.go b/gateway/wrapper/s3config_test.go new file mode 100644 index 000000000..599a3f876 --- /dev/null +++ b/gateway/wrapper/s3config_test.go @@ -0,0 +1,603 @@ +// Tests for conditional logs_store.object_storage injection. +// +// These pin the T1 contract: +// (a) bucket set → logs_store.object_storage emitted with env. +// credential refs, no plaintext key, log_retention_days == 36500 +// (b) bucket unset → output byte-identical to the seed +// (c) hostile bucket/prefix is rejected, not written raw +// (d) second boot with unchanged env is a no-op (no mtime bump) +// +// No Bifrost, no Docker, no network. Run with: +// +// go test ./... +package main + +import ( + "bytes" + "encoding/json" + "log" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +const testSeed = `{ + "$schema": "https://www.getbifrost.ai/schema", + "client": { + "log_retention_days": 36500, + "drop_excess_requests": false, + "enforce_auth_on_inference": true + }, + "auth_config": { + "admin_username": "env.BIFROST_ADMIN_USER", + "admin_password": "env.BIFROST_ADMIN_PASS" + }, + "config_store": { + "enabled": true, + "type": "sqlite", + "config": { + "path": "/app/data/config.db" + } + } +} +` + +const ( + testBucket = "stakgraph-llm-logs" + testRegion = "us-east-1" + testPrefix = "bifrost/org_acme" + testPlainKey = "AKIAIOSFODNN7EXAMPLE" + testPlainSecret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +) + +func TestMaterializeConfig_BucketUnset_ByteIdenticalToSeed(t *testing.T) { + out, info, err := materializeConfigFrom([]byte(testSeed), "/app/data", s3OffloadInfo{}) + if err != nil { + t.Fatalf("materialize: %v", err) + } + if info.Enabled { + t.Fatal("expected S3 offload disabled") + } + if !bytes.Equal(out, []byte(testSeed)) { + t.Fatalf("output must be byte-identical to seed when bucket unset\ngot:\n%s", out) + } +} + +func TestMaterializeConfig_BucketSet_InjectsLogsStore(t *testing.T) { + info := s3OffloadInfo{ + Enabled: true, + Bucket: testBucket, + Region: testRegion, + Prefix: testPrefix, + HasAccessKey: true, + HasSecretKey: true, + } + out, got, err := materializeConfigFrom([]byte(testSeed), "/app/data", info) + if err != nil { + t.Fatalf("materialize: %v", err) + } + if !got.Enabled { + t.Fatal("expected S3 offload enabled") + } + + var parsed map[string]any + if err := json.Unmarshal(out, &parsed); err != nil { + t.Fatalf("output is not valid JSON: %v\n%s", err, out) + } + + client, _ := parsed["client"].(map[string]any) + if client == nil { + t.Fatal("missing client") + } + if days, _ := client["log_retention_days"].(float64); days != 36500 { + t.Fatalf("log_retention_days = %v, want 36500", client["log_retention_days"]) + } + + ls, _ := parsed["logs_store"].(map[string]any) + if ls == nil { + t.Fatalf("missing logs_store:\n%s", out) + } + if ls["enabled"] != true { + t.Errorf("logs_store.enabled = %v, want true", ls["enabled"]) + } + if ls["type"] != "sqlite" { + t.Errorf("logs_store.type = %v, want sqlite", ls["type"]) + } + cfg, _ := ls["config"].(map[string]any) + if cfg["path"] != "/app/data/logs.db" { + t.Errorf("logs_store.config.path = %v, want /app/data/logs.db", cfg["path"]) + } + if _, ok := ls["retention_days"]; ok { + t.Fatal("logs_store must not carry a competing retention_days field") + } + + obj, _ := ls["object_storage"].(map[string]any) + if obj == nil { + t.Fatal("missing object_storage") + } + if obj["type"] != "s3" { + t.Errorf("object_storage.type = %v, want s3", obj["type"]) + } + if obj["bucket"] != testBucket { + t.Errorf("bucket = %v, want %s", obj["bucket"], testBucket) + } + if obj["region"] != testRegion { + t.Errorf("region = %v, want %s", obj["region"], testRegion) + } + if obj["prefix"] != testPrefix { + t.Errorf("prefix = %v, want %s", obj["prefix"], testPrefix) + } + if obj["access_key_id"] != envRefAccessKey { + t.Errorf("access_key_id = %v, want %s", obj["access_key_id"], envRefAccessKey) + } + if obj["secret_access_key"] != envRefSecretKey { + t.Errorf("secret_access_key = %v, want %s", obj["secret_access_key"], envRefSecretKey) + } + if _, ok := obj["compress"]; ok { + t.Fatal("compress must not be set (ship uncompressed until read-back is proven)") + } + + if bytes.Contains(out, []byte(testPlainKey)) || bytes.Contains(out, []byte(testPlainSecret)) { + t.Fatal("plaintext AWS key material must never appear in the written config") + } + if !bytes.Contains(out, []byte(envRefAccessKey)) || !bytes.Contains(out, []byte(envRefSecretKey)) { + t.Fatal("credentials must be emitted as env. references") + } +} + +func TestMaterializeConfig_NoStaticCreds_OmitsKeyFields(t *testing.T) { + info := s3OffloadInfo{ + Enabled: true, + Bucket: testBucket, + Region: testRegion, + Prefix: defaultS3Prefix, + } + out, _, err := materializeConfigFrom([]byte(testSeed), "/app/data", info) + if err != nil { + t.Fatalf("materialize: %v", err) + } + var parsed map[string]any + if err := json.Unmarshal(out, &parsed); err != nil { + t.Fatalf("json: %v", err) + } + obj := parsed["logs_store"].(map[string]any)["object_storage"].(map[string]any) + if _, ok := obj["access_key_id"]; ok { + t.Fatal("access_key_id must be omitted when static creds are absent (instance-role path)") + } + if _, ok := obj["secret_access_key"]; ok { + t.Fatal("secret_access_key must be omitted when static creds are absent") + } +} + +func TestMaterializeConfig_HalfCreds_OmitsKeyFields(t *testing.T) { + // Bifrost rejects a half-configured static-credential block at boot. + info := s3OffloadInfo{ + Enabled: true, + Bucket: testBucket, + Region: testRegion, + Prefix: defaultS3Prefix, + HasAccessKey: true, + HasSecretKey: false, + } + out, _, err := materializeConfigFrom([]byte(testSeed), "/app/data", info) + if err != nil { + t.Fatalf("materialize: %v", err) + } + if bytes.Contains(out, []byte("access_key_id")) { + t.Fatal("half-configured static creds must not emit access_key_id") + } +} + +func TestMaterializeConfig_DefaultPrefix(t *testing.T) { + info := s3OffloadInfo{ + Enabled: true, + Bucket: testBucket, + Region: testRegion, + Prefix: defaultS3Prefix, + } + out, _, err := materializeConfigFrom([]byte(testSeed), "/app/data", info) + if err != nil { + t.Fatalf("materialize: %v", err) + } + var parsed map[string]any + _ = json.Unmarshal(out, &parsed) + obj := parsed["logs_store"].(map[string]any)["object_storage"].(map[string]any) + if obj["prefix"] != defaultS3Prefix { + t.Fatalf("prefix = %v, want %s", obj["prefix"], defaultS3Prefix) + } +} + +func TestMaterializeConfig_EndpointAndPathStyle(t *testing.T) { + info := s3OffloadInfo{ + Enabled: true, + Bucket: testBucket, + Region: testRegion, + Prefix: defaultS3Prefix, + Endpoint: "http://localhost:4566", + ForcePathStyle: true, + HasAccessKey: true, + HasSecretKey: true, + } + out, _, err := materializeConfigFrom([]byte(testSeed), "/app/data", info) + if err != nil { + t.Fatalf("materialize: %v", err) + } + var parsed map[string]any + _ = json.Unmarshal(out, &parsed) + obj := parsed["logs_store"].(map[string]any)["object_storage"].(map[string]any) + if obj["endpoint"] != "http://localhost:4566" { + t.Errorf("endpoint = %v", obj["endpoint"]) + } + if obj["force_path_style"] != true { + t.Errorf("force_path_style = %v, want true", obj["force_path_style"]) + } +} + +func TestMaterializeConfig_Deterministic(t *testing.T) { + info := s3OffloadInfo{ + Enabled: true, + Bucket: testBucket, + Region: testRegion, + Prefix: testPrefix, + HasAccessKey: true, + HasSecretKey: true, + } + a, _, err := materializeConfigFrom([]byte(testSeed), "/app/data", info) + if err != nil { + t.Fatal(err) + } + b, _, err := materializeConfigFrom([]byte(testSeed), "/app/data", info) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(a, b) { + t.Fatal("materializeConfigFrom must be deterministic (idempotency depends on it)") + } +} + +func TestValidate_HostileBucketRejected(t *testing.T) { + hostile := []string{ + `", "injected_key":`, + `foo", "injected_key": "x`, + `../../etc/passwd`, + `Bucket With Spaces`, + `UPPERCASE`, + `ab`, // too short + "", + } + for _, b := range hostile { + info := s3OffloadInfo{Enabled: true, Bucket: b, Region: testRegion, Prefix: defaultS3Prefix} + _, _, err := materializeConfigFrom([]byte(testSeed), "/app/data", info) + if err == nil { + t.Errorf("bucket %q: want validation error, got nil", b) + } + } +} + +func TestValidate_HostilePrefixRejected(t *testing.T) { + hostile := []string{ + `", "injected_key":`, + `foo/../bar`, + `has space`, + `{evil}`, + } + for _, p := range hostile { + info := s3OffloadInfo{Enabled: true, Bucket: testBucket, Region: testRegion, Prefix: p} + _, _, err := materializeConfigFrom([]byte(testSeed), "/app/data", info) + if err == nil { + t.Errorf("prefix %q: want validation error, got nil", p) + } + } +} + +func TestValidate_HostileRegionRejected(t *testing.T) { + info := s3OffloadInfo{Enabled: true, Bucket: testBucket, Region: `us-east-1","x":1`, Prefix: defaultS3Prefix} + if _, _, err := materializeConfigFrom([]byte(testSeed), "/app/data", info); err == nil { + t.Fatal("hostile region must be rejected") + } +} + +func TestJSONMarshalEscapesEvenIfValidationBypassed(t *testing.T) { + // Defense in depth: encoding/json must quote a crafted bucket so it + // cannot close the string and inject a sibling key. + block := objectStorageBlock{ + Type: "s3", + Bucket: `foo", "injected_key": "pwned`, + Region: testRegion, + Prefix: defaultS3Prefix, + } + raw, err := json.Marshal(block) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(raw, []byte(`"injected_key"`)) && !bytes.Contains(raw, []byte(`\"injected_key\"`)) && !bytes.Contains(raw, []byte(`\u0022injected_key`)) { + // A raw (unescaped) injected_key key would mean concatenation won. + var parsed map[string]any + if err := json.Unmarshal(raw, &parsed); err != nil { + t.Fatalf("marshal produced invalid JSON: %v", err) + } + if _, ok := parsed["injected_key"]; ok { + t.Fatalf("hostile bucket injected a sibling key: %s", raw) + } + } + var parsed map[string]any + if err := json.Unmarshal(raw, &parsed); err != nil { + t.Fatalf("marshal produced invalid JSON: %v", err) + } + if _, ok := parsed["injected_key"]; ok { + t.Fatalf("hostile bucket injected a sibling key: %s", raw) + } + if parsed["bucket"] != block.Bucket { + t.Fatalf("bucket round-trip = %v", parsed["bucket"]) + } +} + +func TestReadS3OffloadEnv(t *testing.T) { + t.Setenv(envS3Bucket, "") + t.Setenv(envS3Region, "") + t.Setenv(envS3Prefix, "") + t.Setenv(envS3AccessKey, "") + t.Setenv(envS3SecretKey, "") + t.Setenv(envS3Endpoint, "") + t.Setenv(envS3ForcePathStyle, "") + info := readS3OffloadEnv() + if info.Enabled { + t.Fatal("empty bucket must disable offload") + } + + t.Setenv(envS3Bucket, " "+testBucket+" ") + t.Setenv(envS3Region, testRegion) + t.Setenv(envS3Prefix, testPrefix) + t.Setenv(envS3AccessKey, testPlainKey) + t.Setenv(envS3SecretKey, testPlainSecret) + info = readS3OffloadEnv() + if !info.Enabled || info.Bucket != testBucket || info.Region != testRegion || info.Prefix != testPrefix { + t.Fatalf("unexpected info: %+v", info) + } + if !info.HasAccessKey || !info.HasSecretKey { + t.Fatal("static creds should be detected") + } + if info.HasAccessKey && strings.Contains(info.Bucket, testPlainKey) { + t.Fatal("info must not carry key material on Bucket") + } +} + +func TestMaterializeConfig_RealSeed(t *testing.T) { + seedPath := filepath.Join("..", "data", "config.json") + seed, err := os.ReadFile(seedPath) + if err != nil { + t.Skipf("seed not available: %v", err) + } + + off, _, err := materializeConfigFrom(seed, "/app/data", s3OffloadInfo{}) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(off, seed) { + t.Fatal("real seed must pass through byte-identical when S3 is off") + } + + on, _, err := materializeConfigFrom(seed, "/app/data", s3OffloadInfo{ + Enabled: true, + Bucket: testBucket, + Region: testRegion, + Prefix: testPrefix, + HasAccessKey: true, + HasSecretKey: true, + }) + if err != nil { + t.Fatal(err) + } + var parsed map[string]any + if err := json.Unmarshal(on, &parsed); err != nil { + t.Fatalf("injected real seed is not JSON: %v", err) + } + if parsed["plugins"] == nil || parsed["providers"] == nil || parsed["auth_config"] == nil { + t.Fatal("injection must preserve plugins/providers/auth_config") + } + ls := parsed["logs_store"].(map[string]any) + obj := ls["object_storage"].(map[string]any) + if obj["access_key_id"] != envRefAccessKey { + t.Fatal("real-seed injection must use env.* credential refs") + } + if bytes.Contains(on, []byte(testPlainKey)) { + t.Fatal("plaintext key in real-seed output") + } +} + +func TestSeedFile_LogRetentionDays(t *testing.T) { + seedPath := filepath.Join("..", "data", "config.json") + raw, err := os.ReadFile(seedPath) + if err != nil { + t.Skipf("seed not available: %v", err) + } + var parsed map[string]any + if err := json.Unmarshal(raw, &parsed); err != nil { + t.Fatalf("seed is not valid JSON: %v", err) + } + client, _ := parsed["client"].(map[string]any) + if days, _ := client["log_retention_days"].(float64); days != 36500 { + t.Fatalf("data/config.json client.log_retention_days = %v, want 36500", client["log_retention_days"]) + } + if _, ok := parsed["logs_store"]; ok { + t.Fatal("seed must not contain a logs_store block (injection is opt-in at boot)") + } +} + +func TestSyncSeedConfig_S3Off_ByteIdenticalAndIdempotent(t *testing.T) { + t.Setenv(envS3Bucket, "") + t.Setenv(envS3Region, "") + t.Setenv(envS3Prefix, "") + t.Setenv(envS3AccessKey, "") + t.Setenv(envS3SecretKey, "") + + dir := t.TempDir() + seedPath := filepath.Join(dir, "config.json.seed") + appDir := filepath.Join(dir, "data") + if err := os.WriteFile(seedPath, []byte(testSeed), 0o644); err != nil { + t.Fatal(err) + } + logger := log.New(os.Stderr, "[test] ", 0) + if err := syncSeedConfig(logger, seedPath, appDir); err != nil { + t.Fatalf("first sync: %v", err) + } + got, err := os.ReadFile(filepath.Join(appDir, "config.json")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, []byte(testSeed)) { + t.Fatalf("S3-off output must equal seed\ngot:\n%s", got) + } + + dst := filepath.Join(appDir, "config.json") + past := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) + if err := os.Chtimes(dst, past, past); err != nil { + t.Fatal(err) + } + if err := syncSeedConfig(logger, seedPath, appDir); err != nil { + t.Fatalf("second sync: %v", err) + } + st, err := os.Stat(dst) + if err != nil { + t.Fatal(err) + } + if !st.ModTime().Equal(past) { + t.Fatalf("idempotent S3-off sync bumped mtime: got %v want %v", st.ModTime(), past) + } +} + +func TestSyncSeedConfig_S3On_EnvRefsNoPlaintextIdempotent(t *testing.T) { + t.Setenv(envS3Bucket, testBucket) + t.Setenv(envS3Region, testRegion) + t.Setenv(envS3Prefix, testPrefix) + t.Setenv(envS3AccessKey, testPlainKey) + t.Setenv(envS3SecretKey, testPlainSecret) + + dir := t.TempDir() + seedPath := filepath.Join(dir, "config.json.seed") + appDir := filepath.Join(dir, "data") + if err := os.WriteFile(seedPath, []byte(testSeed), 0o644); err != nil { + t.Fatal(err) + } + logger := log.New(os.Stderr, "[test] ", 0) + if err := syncSeedConfig(logger, seedPath, appDir); err != nil { + t.Fatalf("first sync: %v", err) + } + + dst := filepath.Join(appDir, "config.json") + got, err := os.ReadFile(dst) + if err != nil { + t.Fatal(err) + } + if bytes.Equal(got, []byte(testSeed)) { + t.Fatal("S3-on output must differ from seed (logs_store injected)") + } + if bytes.Contains(got, []byte(testPlainKey)) || bytes.Contains(got, []byte(testPlainSecret)) { + t.Fatal("plaintext key material leaked into config.json") + } + if !bytes.Contains(got, []byte(envRefAccessKey)) || !bytes.Contains(got, []byte(envRefSecretKey)) { + t.Fatal("missing env. credential references") + } + if !bytes.Contains(got, []byte(`"bucket": "`+testBucket+`"`)) { + t.Fatalf("missing bucket in output:\n%s", got) + } + + var parsed map[string]any + if err := json.Unmarshal(got, &parsed); err != nil { + t.Fatalf("written config is not JSON: %v", err) + } + client := parsed["client"].(map[string]any) + if days, _ := client["log_retention_days"].(float64); days != 36500 { + t.Fatalf("log_retention_days = %v, want 36500", client["log_retention_days"]) + } + + past := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) + if err := os.Chtimes(dst, past, past); err != nil { + t.Fatal(err) + } + if err := syncSeedConfig(logger, seedPath, appDir); err != nil { + t.Fatalf("second sync: %v", err) + } + st, err := os.Stat(dst) + if err != nil { + t.Fatal(err) + } + if !st.ModTime().Equal(past) { + t.Fatalf("idempotent S3-on sync bumped mtime: got %v want %v", st.ModTime(), past) + } + + // Changing env must rewrite. + t.Setenv(envS3Prefix, "bifrost/other-org") + if err := syncSeedConfig(logger, seedPath, appDir); err != nil { + t.Fatalf("third sync: %v", err) + } + st, err = os.Stat(dst) + if err != nil { + t.Fatal(err) + } + if st.ModTime().Equal(past) { + t.Fatal("changing BIFROST_S3_PREFIX should rewrite config.json") + } + rewritten, _ := os.ReadFile(dst) + if !bytes.Contains(rewritten, []byte("bifrost/other-org")) { + t.Fatalf("rewritten config missing new prefix:\n%s", rewritten) + } +} + +func TestSyncSeedConfig_HostileBucketFallsBackToSeed(t *testing.T) { + t.Setenv(envS3Bucket, `foo", "injected_key": "pwned`) + t.Setenv(envS3Region, testRegion) + t.Setenv(envS3Prefix, defaultS3Prefix) + t.Setenv(envS3AccessKey, testPlainKey) + t.Setenv(envS3SecretKey, testPlainSecret) + + dir := t.TempDir() + seedPath := filepath.Join(dir, "config.json.seed") + appDir := filepath.Join(dir, "data") + if err := os.WriteFile(seedPath, []byte(testSeed), 0o644); err != nil { + t.Fatal(err) + } + logger := log.New(os.Stderr, "[test] ", 0) + if err := syncSeedConfig(logger, seedPath, appDir); err != nil { + t.Fatalf("sync: %v", err) + } + got, err := os.ReadFile(filepath.Join(appDir, "config.json")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, []byte(testSeed)) { + t.Fatalf("hostile bucket must fall back to seed, not write injected JSON\ngot:\n%s", got) + } + if bytes.Contains(got, []byte("injected_key")) { + t.Fatal("injected_key leaked into written config") + } +} + +func TestLogS3Offload_NeverPrintsSecrets(t *testing.T) { + var buf bytes.Buffer + logger := log.New(&buf, "", 0) + logS3Offload(logger, s3OffloadInfo{ + Enabled: true, + Bucket: testBucket, + Region: testRegion, + Prefix: testPrefix, + HasAccessKey: true, + HasSecretKey: true, + }) + s := buf.String() + if !strings.Contains(s, testBucket) || !strings.Contains(s, testRegion) { + t.Fatalf("expected bucket+region in log, got %q", s) + } + if strings.Contains(s, testPlainKey) || strings.Contains(s, testPlainSecret) { + t.Fatalf("secrets leaked in log: %q", s) + } + + buf.Reset() + logS3Offload(logger, s3OffloadInfo{Enabled: true, Bucket: testBucket}) + s = buf.String() + if !strings.Contains(s, "WARNING") { + t.Fatalf("missing WARNING for bucket-without-region/creds: %q", s) + } +}