diff --git a/CLAUDE.md b/CLAUDE.md index 467edb3..f8c8b27 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,92 +1,94 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Project Overview - -LAPP (Log Auto Pattern Pipeline) discovers log templates from log streams using the Drain algorithm, labels them with semantic IDs via LLM, and builds structured file-based workspaces for AI-assisted log investigation. It also includes an agentic analyzer that uses LLMs to investigate logs. - -## Commands - -```bash -make build # Build embedded frontend assets, then output/lapp -make clean # Remove generated build artifacts -make dev # Clean, build, and start lapp web on 127.0.0.1:8080 -make proto-gen # Generate protobuf/Connect code -make test # Run unit and integration tests -make check # Run formatting, linting, type checks, build, and unit tests - -# Run a single test -go test -v -run TestFunctionName ./pkg/pattern/ -``` - -## CLI Usage - -```bash -go run ./cmd/lapp/ workspace create -go run ./cmd/lapp/ workspace add-log --topic -go run ./cmd/lapp/ workspace add-log --topic --stdin -go run ./cmd/lapp/ workspace discover --topic [--model ] -go run ./cmd/lapp/ workspace analyze --topic [question] [--model ] -go run ./cmd/lapp/ web [--addr 127.0.0.1:0] -``` - -Topic names are sanitized to lower-kebab-case. Workspaces live under `~/.lapp/workspaces//`. - -## Architecture - -``` -cmd/lapp/ CLI entrypoint (cobra commands: workspace create/add-log/discover/analyze) -pkg/logsource/ Read log files → channel of LogLine -pkg/multiline/ Detect log entry boundaries, merge continuation lines -pkg/pattern/ Drain-based log pattern discovery and template matching -pkg/semantic/ LLM-based semantic labeling of Drain patterns -pkg/workspace/ DiscoveryRun execution and run-scoped file writer -pkg/store/ DuckDB storage primitives (not yet on the CLI discovery path) -pkg/config/ Model resolution (flag → $MODEL_NAME → default) -pkg/analyzer/ Agentic log analysis via eino ADK + ACP providers -integration_test/ Integration tests against Loghub-2.0 datasets -``` - -### DiscoveryRun (discover) - -`add-log` is a pure copy into `logs/` and never triggers discovery. Each `workspace discover` starts a DiscoveryRun: reads ALL files in `logs/`, runs fresh Drain + semantic labeling, and writes run-scoped `patterns/` and `notes/`. -When `lapp web` starts, it marks any previous `QUEUED` or `RUNNING` DiscoveryRuns as failed because those local workers no longer exist. -DiscoveryRun records persist structured `progress` and `error` fields; frontend code renders those facts into user-facing text. - -``` -workspace.Discover(ctx, cfg) - → Read all logs/ files → multiline.MergeSlice() per file → tagged lines - → pattern.DrainParser.Feed(all content) → Templates() → filter Count > 1 - → semantic.Label(ctx, cfg, patterns) ← LLM batches with per-batch retry - → workspace.NewBuilder(...).BuildAll() - → discovery-runs//patterns//pattern.md + samples.log - → discovery-runs//patterns/unmatched/samples.log - → discovery-runs//notes/summary.md + errors.md - → discovery-runs//AGENTS.md -``` - -### Multiline Detection - -Uses a token graph trained on 70+ timestamp formats. Lines are tokenized (first 60 bytes), matched against a directed graph of valid token transitions, and scored 0.0-1.0. Score > 0.5 means "new log entry". If no timestamps are ever detected, falls back to line-by-line. - -### Analyzer - -Runs an eino ADK agent (15 max iterations) with filesystem tools (grep, read_file, execute) against a structured workspace directory. - -## Environment Variables - -- `OPENROUTER_API_KEY`: Required for semantic labeling in `workspace discover` -- `MODEL_NAME`: Override default LLM model (default: `google/gemini-3-flash-preview`) -- ACP provider credentials/login: Required for `workspace analyze` through the selected provider -- `.env` file is auto-loaded via godotenv - -## Tech Stack - -- Go, cobra CLI, go-drain3, DuckDB (duckdb-go/v2), cloudwego/eino ADK, OpenRouter semantic labeling, ACP providers - -## Code Style - -- `nolint` directives go on the line above the target, not as end-of-line comments -- Compile-time interface guards: `var _ MyInterface = (*MyImpl)(nil)` -- Always use `make build` to verify compilation, never bare `go build` (it drops a binary in the project root) +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +LAPP (Log Auto Pattern Pipeline) discovers log templates from log streams using the Drain algorithm, labels them with semantic IDs via LLM, and builds structured file-based workspaces for AI-assisted log investigation. It also includes an agentic analyzer that uses LLMs to investigate logs. + +## Commands + +```bash +make build # Build embedded frontend assets, then output/lapp +make clean # Remove generated build artifacts +make dev # Clean, build, and start lapp web on 127.0.0.1:8080 +make proto-gen # Generate protobuf/Connect code +make test # Run unit and integration tests +make check # Run formatting, linting, type checks, build, and unit tests + +# Run a single test +go test -v -run TestFunctionName ./pkg/pattern/ +``` + +## CLI Usage + +```bash +go run ./cmd/lapp/ workspace create +go run ./cmd/lapp/ workspace add-log --topic +go run ./cmd/lapp/ workspace add-log --topic --stdin +go run ./cmd/lapp/ workspace import gcp --topic --project [--filter ] (--since | --from --to ) [--limit ] +go run ./cmd/lapp/ workspace discover --topic [--model ] +go run ./cmd/lapp/ workspace analyze --topic [question] [--model ] +go run ./cmd/lapp/ web [--addr 127.0.0.1:0] +``` + +Topic names are sanitized to lower-kebab-case. Workspaces live under `~/.lapp/workspaces//`. + +## Architecture + +``` +cmd/lapp/ CLI entrypoint (cobra commands: workspace create/add-log/import/discover/analyze) +pkg/logsource/ Read log files → channel of LogLine +pkg/multiline/ Detect log entry boundaries, merge continuation lines +pkg/pattern/ Drain-based log pattern discovery and template matching +pkg/semantic/ LLM-based semantic labeling of Drain patterns +pkg/workspace/ DiscoveryRun execution and run-scoped file writer +pkg/store/ DuckDB storage primitives (not yet on the CLI discovery path) +pkg/config/ Model resolution (flag → $MODEL_NAME → default) +pkg/analyzer/ Agentic log analysis via eino ADK + ACP providers +integration_test/ Integration tests against Loghub-2.0 datasets +``` + +### DiscoveryRun (discover) + +`add-log` is a pure copy into `logs/` and never triggers discovery. `workspace import gcp` pulls a snapshot from GCP Cloud Logging through ADC credentials, lands it as enveloped NDJSON in `logs/`, and records provenance under `import-runs//record.json`; it never triggers discovery either. Each `workspace discover` starts a DiscoveryRun: reads ALL files in `logs/`, runs fresh Drain + semantic labeling, and writes run-scoped `patterns/` and `notes/`. +When `lapp web` starts, it marks any previous `QUEUED` or `RUNNING` DiscoveryRuns as failed because those local workers no longer exist. +DiscoveryRun records persist structured `progress` and `error` fields; frontend code renders those facts into user-facing text. + +``` +workspace.Discover(ctx, cfg) + → Read all logs/ files → multiline.MergeSlice() per file → tagged lines + → pattern.DrainParser.Feed(all content) → Templates() → filter Count > 1 + → semantic.Label(ctx, cfg, patterns) ← LLM batches with per-batch retry + → workspace.NewBuilder(...).BuildAll() + → discovery-runs//patterns//pattern.md + samples.log + → discovery-runs//patterns/unmatched/samples.log + → discovery-runs//notes/summary.md + errors.md + → discovery-runs//AGENTS.md +``` + +### Multiline Detection + +Uses a token graph trained on 70+ timestamp formats. Lines are tokenized (first 60 bytes), matched against a directed graph of valid token transitions, and scored 0.0-1.0. Score > 0.5 means "new log entry". If no timestamps are ever detected, falls back to line-by-line. + +### Analyzer + +Runs an eino ADK agent (15 max iterations) with filesystem tools (grep, read_file, execute) against a structured workspace directory. + +## Environment Variables + +- `OPENROUTER_API_KEY`: Required for semantic labeling in `workspace discover` +- GCP ADC (`gcloud auth application-default login`): Required for `workspace import gcp` +- `MODEL_NAME`: Override default LLM model (default: `google/gemini-3-flash-preview`) +- ACP provider credentials/login: Required for `workspace analyze` through the selected provider +- `.env` file is auto-loaded via godotenv + +## Tech Stack + +- Go, cobra CLI, go-drain3, DuckDB (duckdb-go/v2), cloudwego/eino ADK, OpenRouter semantic labeling, ACP providers + +## Code Style + +- `nolint` directives go on the line above the target, not as end-of-line comments +- Compile-time interface guards: `var _ MyInterface = (*MyImpl)(nil)` +- Always use `make build` to verify compilation, never bare `go build` (it drops a binary in the project root) diff --git a/README.md b/README.md index 4c66620..77aff6d 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,7 @@ DiscoveryRuns are one-time local tasks. If `lapp web` starts and finds a previou | `workspace create ` | Create a workspace under `~/.lapp/workspaces/` | | `workspace list` | List all workspace topics | | `workspace add-log --topic ` | Copy a log file into the workspace | +| `workspace import gcp --topic --project

--since 1h` | Import a snapshot from GCP Cloud Logging | | `workspace discover --topic ` | Run pattern discovery over all log files | | `workspace analyze --topic [question]` | Run AI analysis (`--acp claude|codex|gemini`) | diff --git a/cmd/lapp/workspace.go b/cmd/lapp/workspace.go index 72e5182..42d87ee 100644 --- a/cmd/lapp/workspace.go +++ b/cmd/lapp/workspace.go @@ -46,6 +46,7 @@ func workspaceCmd() *cobra.Command { cmd.AddCommand(workspaceCreateCmd()) cmd.AddCommand(workspaceListCmd()) cmd.AddCommand(workspaceAddLogCmd()) + cmd.AddCommand(workspaceImportCmd()) cmd.AddCommand(workspaceDiscoverCmd()) cmd.AddCommand(workspaceAnalyzeCmd()) return cmd @@ -126,7 +127,7 @@ func runWorkspaceCreate(_ *cobra.Command, args []string) error { return err } - for _, sub := range []string{"logs", workspace.DiscoveryRunsDirName} { + for _, sub := range []string{"logs", workspace.DiscoveryRunsDirName, workspace.ImportRunsDirName} { if err := os.MkdirAll(filepath.Join(dir, sub), 0o755); err != nil { return errors.Errorf("create %s: %w", sub, err) } diff --git a/cmd/lapp/workspace_import.go b/cmd/lapp/workspace_import.go new file mode 100644 index 0000000..1b7794b --- /dev/null +++ b/cmd/lapp/workspace_import.go @@ -0,0 +1,177 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/go-errors/errors" + "github.com/spf13/cobra" + "github.com/strrl/lapp/pkg/gcplog" + "github.com/strrl/lapp/pkg/workspace" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/codes" +) + +var importGCPTopic string +var importGCPProject string +var importGCPFilter string +var importGCPSince string +var importGCPFrom string +var importGCPTo string +var importGCPLimit int + +func workspaceImportCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "import", + Short: "Import logs from an external provider into the workspace", + } + cmd.AddCommand(workspaceImportGCPCmd()) + return cmd +} + +func workspaceImportGCPCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "gcp", + Short: "Import logs from GCP Cloud Logging", + Long: `Pull matching entries from GCP Cloud Logging into the workspace as one +NDJSON log file under logs/, and record the ImportRun under import-runs/. + +Uses Application Default Credentials; LAPP never manages provider auth. +Exactly one time range form is required: --since, or --from together with --to. +This never starts discovery. Run 'lapp workspace discover --topic ' +afterwards.`, + Args: cobra.NoArgs, + RunE: runWorkspaceImportGCP, + } + cmd.Flags().StringVar(&importGCPTopic, "topic", "", "workspace topic (required)") + cmd.Flags().StringVar(&importGCPProject, "project", "", "GCP project id (required)") + cmd.Flags().StringVar(&importGCPFilter, "filter", "", "Cloud Logging filter, combined with the time range") + cmd.Flags().StringVar(&importGCPSince, "since", "", "import entries from the last duration, e.g. 1h or 30m") + cmd.Flags().StringVar(&importGCPFrom, "from", "", "start of time range, RFC3339, e.g. 2026-07-25T00:00:00Z") + cmd.Flags().StringVar(&importGCPTo, "to", "", "end of time range, RFC3339") + cmd.Flags().IntVar(&importGCPLimit, "limit", 100000, "maximum number of entries to import") + _ = cmd.MarkFlagRequired("topic") + _ = cmd.MarkFlagRequired("project") + return cmd +} + +// resolveImportTimeRange validates the mutually exclusive time range flags +// and resolves them to a concrete UTC from/to pair. +func resolveImportTimeRange(since, from, to string, now time.Time) (fromTime, toTime time.Time, err error) { + hasSince := since != "" + hasFromTo := from != "" || to != "" + if hasSince && hasFromTo { + return time.Time{}, time.Time{}, errors.New("--since and --from/--to are mutually exclusive; use one time range form") + } + if !hasSince && !hasFromTo { + return time.Time{}, time.Time{}, errors.New("a time range is required: use --since , or --from and --to (RFC3339)") + } + + if hasSince { + duration, parseErr := time.ParseDuration(since) + if parseErr != nil { + return time.Time{}, time.Time{}, errors.Errorf("invalid --since duration %q: %w", since, parseErr) + } + if duration <= 0 { + return time.Time{}, time.Time{}, errors.Errorf("--since must be a positive duration, got %q", since) + } + toTime = now.UTC() + return toTime.Add(-duration), toTime, nil + } + + if from == "" || to == "" { + return time.Time{}, time.Time{}, errors.New("--from and --to must be provided together") + } + fromTime, err = time.Parse(time.RFC3339, from) + if err != nil { + return time.Time{}, time.Time{}, errors.Errorf("invalid --from timestamp %q, expected RFC3339: %w", from, err) + } + toTime, err = time.Parse(time.RFC3339, to) + if err != nil { + return time.Time{}, time.Time{}, errors.Errorf("invalid --to timestamp %q, expected RFC3339: %w", to, err) + } + if fromTime.After(toTime) { + return time.Time{}, time.Time{}, errors.Errorf("--from %s is after --to %s", from, to) + } + return fromTime.UTC(), toTime.UTC(), nil +} + +func runWorkspaceImportGCP(cmd *cobra.Command, _ []string) error { + dir, err := topicToDir(importGCPTopic) + if err != nil { + return err + } + + // Validate workspace exists + if _, err := os.Stat(filepath.Join(dir, "logs")); os.IsNotExist(err) { + hint := availableWorkspacesHint() + return errors.Errorf("not a workspace: %s (no logs/ directory)%s", dir, hint) + } + + if strings.TrimSpace(importGCPProject) == "" { + return errors.New("--project must not be blank") + } + + from, to, err := resolveImportTimeRange(importGCPSince, importGCPFrom, importGCPTo, time.Now()) + if err != nil { + return err + } + if importGCPLimit <= 0 { + return errors.Errorf("--limit must be positive, got %d", importGCPLimit) + } + + // Validation is done; anything past this point is a run failure, so the + // usage help would only be noise. + cmd.SilenceUsage = true + + ctx, span := otel.Tracer("lapp/cmd").Start(cmd.Context(), "cmd.WorkspaceImportGCP") + defer span.End() + + result, err := workspace.RunImport(ctx, workspace.ImportConfig{ + Dir: dir, + Provider: "gcp", + Project: importGCPProject, + Filter: importGCPFilter, + From: from, + To: to, + Limit: importGCPLimit, + Fetcher: func(fetchCtx context.Context, req workspace.ImportRequest) (workspace.ImportFetchResult, error) { + fetched, err := gcplog.FetchLines(fetchCtx, gcplog.FetchRequest{ + Project: req.Project, + Filter: req.Filter, + From: req.From, + To: req.To, + Limit: req.Limit, + }) + if err != nil { + return workspace.ImportFetchResult{}, err + } + return workspace.ImportFetchResult{ + Lines: fetched.Lines, + Truncated: fetched.Truncated, + }, nil + }, + }) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return err + } + + topic := filepath.Base(dir) + if result.EntryCount == 0 { + fmt.Printf("No entries matched. ImportRun %s succeeded with 0 entries; no log file was written.\n", result.RunID) + span.SetStatus(codes.Ok, "") + return nil + } + if result.Truncated { + fmt.Printf("Warning: result truncated at --limit %d; narrow the time range or filter to import everything.\n", importGCPLimit) + } + fmt.Printf("Imported %d entries into logs/%s.\nDiscovery has not run yet. Run it with:\n\n lapp workspace discover --topic %s\n", result.EntryCount, result.LogFileName, topic) + span.SetStatus(codes.Ok, "") + return nil +} diff --git a/go.mod b/go.mod index 1a43afd..32c6055 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/strrl/lapp go 1.25.7 require ( + cloud.google.com/go/logging v1.19.0 connectrpc.com/connect v1.20.0 github.com/bytedance/sonic v1.15.0 github.com/cloudwego/eino v0.8.0 @@ -15,14 +16,21 @@ require ( github.com/joho/godotenv v1.5.1 github.com/spf13/cobra v1.10.2 github.com/strrl/eino-acp v0.0.0-20260320032654-943782f485e5 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 - go.opentelemetry.io/otel v1.40.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 + go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 - go.opentelemetry.io/otel/sdk v1.40.0 + go.opentelemetry.io/otel/sdk v1.44.0 + google.golang.org/api v0.290.0 google.golang.org/protobuf v1.36.11 ) require ( + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.11.0 // indirect + cloud.google.com/go/longrunning v1.2.0 // indirect github.com/apache/arrow-go/v18 v18.5.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect @@ -50,6 +58,9 @@ require ( github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/google/flatbuffers v25.12.19+incompatible // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.18 // indirect + github.com/googleapis/gax-go/v2 v2.23.0 // indirect github.com/goph/emperror v0.17.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect @@ -77,22 +88,27 @@ require ( github.com/yargevad/filepathx v1.0.0 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect - go.opentelemetry.io/otel/metric v1.40.0 // indirect - go.opentelemetry.io/otel/trace v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect golang.org/x/arch v0.12.0 // indirect + golang.org/x/crypto v0.54.0 // indirect golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect - golang.org/x/mod v0.32.0 // indirect - golang.org/x/net v0.49.0 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.40.0 // indirect - golang.org/x/telemetry v0.0.0-20260116145544-c6413dc483f5 // indirect - golang.org/x/text v0.33.0 // indirect - golang.org/x/tools v0.41.0 // indirect + golang.org/x/mod v0.37.0 // indirect + golang.org/x/net v0.57.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 // indirect + golang.org/x/text v0.40.0 // indirect + golang.org/x/time v0.15.0 // indirect + golang.org/x/tools v0.47.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect - google.golang.org/grpc v1.78.0 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect + google.golang.org/grpc v1.82.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 6ddc462..7a16eb7 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,31 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= +cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= +cloud.google.com/go/logging v1.19.0 h1:NCqhdVUg3wQ8Cobdf16FDSuTGi3+6+hdSBHrY5TsR6Q= +cloud.google.com/go/logging v1.19.0/go.mod h1:i40NZCHC9Gqvod4yE+yQfDWwlgwW/SrshkkGibCHxcA= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/storage v1.62.3 h1:SZq1t23NCI+e96dH77Dg3PEfsNNEjqO8zE5AnD8gVD0= +cloud.google.com/go/storage v1.62.3/go.mod h1:cpYz/kRVZ+UQAF1uHeea10/9ewcRbxGoGNKsS9daSXA= connectrpc.com/connect v1.20.0 h1:6TNDAB+WeNd2uolWNlYczB5E0KNNaVMNUEx8JEUsPmQ= connectrpc.com/connect v1.20.0/go.mod h1:A2ygJrukXwWy32vkCAAHNVguZrqZ+jeZ9rGRnGR4dN4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= @@ -42,6 +68,8 @@ github.com/cloudwego/eino-ext/libs/acl/langfuse v0.0.0-20251124083837-ce2e7e196f github.com/cloudwego/eino-ext/libs/acl/langfuse v0.0.0-20251124083837-ce2e7e196f9f/go.mod h1:P3zzJTRexY0QKaE9Vn2CmOnCorIMgNzNtler8mw9IQM= github.com/cloudwego/eino-ext/libs/acl/openai v0.1.13 h1:z0bI5TH3nE+uDQiRhxBQMvk2HswlDUM3xP38+VSgpSQ= github.com/cloudwego/eino-ext/libs/acl/openai v0.1.13/go.mod h1:1xMQZ8eE11pkEoTAEy8UlaAY817qGVMvjpDPGSIO3Ns= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/coder/acp-go-sdk v0.6.3 h1:LsXQytehdjKIYJnoVWON/nf7mqbiarnyuyE3rrjBsXQ= github.com/coder/acp-go-sdk v0.6.3/go.mod h1:yKzM/3R9uELp4+nBAwwtkS0aN1FOFjo11CNPy37yFko= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -67,6 +95,11 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eino-contrib/jsonschema v1.0.3 h1:2Kfsm1xlMV0ssY2nuxshS4AwbLFuqmPmzIjLVJ1Fsp0= github.com/eino-contrib/jsonschema v1.0.3/go.mod h1:cpnX4SyKjWjGC7iN2EbhxaTdLqGjCi0e9DxpLYxddD4= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -77,6 +110,8 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJY github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -87,6 +122,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -99,8 +136,14 @@ github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZa github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.18 h1:hvVi34VucdrV1IIsiWuqYM8kutw/92MxNEFxCJZEh0k= +github.com/googleapis/enterprise-certificate-proxy v0.3.18/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= +github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= @@ -168,6 +211,8 @@ github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcR github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -188,6 +233,8 @@ github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -224,24 +271,30 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= -go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= -go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 h1:wVZXIWjQSeSmMoxF74LzAnpVQOAFDo3pPji9Y4SOFKc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0/go.mod h1:khvBS2IggMFNwZK/6lEeHg/W57h/IX6J4URh57fuI40= -go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= -go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= -go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= -go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= -go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= -go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= -go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= -go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -252,42 +305,50 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg= golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= -golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= -golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= -golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= -golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/telemetry v0.0.0-20260116145544-c6413dc483f5 h1:i0p03B68+xC1kD2QUO8JzDTPXCzhN56OLJ+IhHY8U3A= -golang.org/x/telemetry v0.0.0-20260116145544-c6413dc483f5/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8= -golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= -golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 h1:nwGZBCt+FnXUrGsj5vjzAsEmkcaFvd82BbOjECiFYZc= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= -golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= -golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= -golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= -google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= -google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= -google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.290.0 h1:eMw0Xo+IfbbMlKmW7aHvpyQRv9RCXuWx/vs8AD+0x9A= +google.golang.org/api v0.290.0/go.mod h1:weJZ3lldHFYI0DBFNKpJelUDNnusTt5YaOEgxvt8ci8= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 h1:qEHAMpSaUhtD0p3NbEEI83HwNGFxEwaSJ1G9PLnCBZE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= +google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/gcplog/convert.go b/pkg/gcplog/convert.go new file mode 100644 index 0000000..85386f5 --- /dev/null +++ b/pkg/gcplog/convert.go @@ -0,0 +1,84 @@ +// Package gcplog pulls log entries from GCP Cloud Logging and converts them +// into the workspace NDJSON envelope defined by ADR 0006. +package gcplog + +import ( + "bytes" + "encoding/json" + "strings" + "time" + + "github.com/go-errors/errors" +) + +// DefaultSeverity is GCP's severity for entries without an explicit level. +const DefaultSeverity = "DEFAULT" + +// Entry is one GCP Cloud Logging entry, using the REST JSON field names. +// At most one of JSONPayload and TextPayload is set. TextPayload is a +// pointer so a present but empty text payload stays distinguishable from an +// absent one. +type Entry struct { + Timestamp time.Time `json:"timestamp"` + Severity string `json:"severity,omitempty"` + JSONPayload json.RawMessage `json:"jsonPayload,omitempty"` + TextPayload *string `json:"textPayload,omitempty"` +} + +// envelope is the fixed NDJSON shape imported entries land in (ADR 0006). +type envelope struct { + TS string `json:"ts"` + Severity string `json:"severity"` + Payload json.RawMessage `json:"payload"` +} + +// ConvertEntry converts one GCP log entry into one envelope NDJSON line +// without a trailing newline. The timestamp is normalized to RFC3339 UTC, +// a missing severity becomes DEFAULT, jsonPayload is kept as is (compacted +// to a single line), and textPayload becomes {"message": }. Provider +// metadata such as labels, trace, and resource is intentionally dropped +// here; it belongs in the ImportRun record, not in the log lines. +func ConvertEntry(entry Entry) (string, error) { + payload, err := entryPayload(entry) + if err != nil { + return "", err + } + severity := entry.Severity + if severity == "" { + severity = DefaultSeverity + } + return marshalEnvelopeLine(envelope{ + TS: entry.Timestamp.UTC().Format(time.RFC3339Nano), + Severity: severity, + Payload: payload, + }) +} + +func entryPayload(entry Entry) (json.RawMessage, error) { + trimmed := bytes.TrimSpace(entry.JSONPayload) + if len(trimmed) > 0 && !bytes.Equal(trimmed, []byte("null")) { + var buf bytes.Buffer + if err := json.Compact(&buf, trimmed); err != nil { + return nil, errors.Errorf("compact jsonPayload: %w", err) + } + return buf.Bytes(), nil + } + if entry.TextPayload != nil { + data, err := json.Marshal(map[string]string{"message": *entry.TextPayload}) + if err != nil { + return nil, errors.Errorf("marshal textPayload: %w", err) + } + return data, nil + } + return json.RawMessage("{}"), nil +} + +func marshalEnvelopeLine(env envelope) (string, error) { + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(env); err != nil { + return "", errors.Errorf("marshal envelope: %w", err) + } + return strings.TrimSuffix(buf.String(), "\n"), nil +} diff --git a/pkg/gcplog/convert_test.go b/pkg/gcplog/convert_test.go new file mode 100644 index 0000000..826cf87 --- /dev/null +++ b/pkg/gcplog/convert_test.go @@ -0,0 +1,81 @@ +package gcplog + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// TestConvertEntryFixtures converts every GCP entry fixture under +// testdata/convert/ and compares the result against its committed expected +// envelope line. Each .input.json pairs with .expected.txt, so +// adding a case only means adding one fixture pair. +func TestConvertEntryFixtures(t *testing.T) { + inputs, err := filepath.Glob(filepath.Join("testdata", "convert", "*.input.json")) + if err != nil { + t.Fatalf("glob convert fixtures: %v", err) + } + if len(inputs) == 0 { + t.Fatal("no convert fixtures found under testdata/convert") + } + + for _, inputPath := range inputs { + name := strings.TrimSuffix(filepath.Base(inputPath), ".input.json") + expectedPath := filepath.Join("testdata", "convert", name+".expected.txt") + t.Run(name, func(t *testing.T) { + data, err := os.ReadFile(inputPath) + if err != nil { + t.Fatalf("read input fixture %s: %v", inputPath, err) + } + var entry Entry + if err := json.Unmarshal(data, &entry); err != nil { + t.Fatalf("decode input fixture %s: %v", inputPath, err) + } + + line, err := ConvertEntry(entry) + if err != nil { + t.Fatalf("ConvertEntry: %v", err) + } + got := line + "\n" + + expected, err := os.ReadFile(expectedPath) + if err != nil { + t.Fatalf("read expected output %s: %v", expectedPath, err) + } + if got != string(expected) { + t.Fatalf("conversion mismatch for %s\ngot:\n%swant:\n%s", inputPath, got, expected) + } + }) + } +} + +func TestCombinedFilterJoinsTimeRangeAndFilter(t *testing.T) { + from := time.Date(2026, 7, 25, 8, 0, 0, 0, time.FixedZone("CST", 8*3600)) + to := time.Date(2026, 7, 25, 9, 0, 0, 0, time.FixedZone("CST", 8*3600)) + + got := CombinedFilter(`resource.type="cloud_run_revision"`, from, to) + want := `timestamp >= "2026-07-25T00:00:00Z" AND timestamp <= "2026-07-25T01:00:00Z" AND (resource.type="cloud_run_revision")` + if got != want { + t.Fatalf("CombinedFilter with filter = %q, want %q", got, want) + } + + got = CombinedFilter("", from, to) + want = `timestamp >= "2026-07-25T00:00:00Z" AND timestamp <= "2026-07-25T01:00:00Z"` + if got != want { + t.Fatalf("CombinedFilter without filter = %q, want %q", got, want) + } +} + +func TestCombinedFilterKeepsFractionalSeconds(t *testing.T) { + from := time.Date(2026, 7, 25, 0, 0, 0, 250_000_000, time.UTC) + to := time.Date(2026, 7, 25, 1, 0, 0, 123_456_789, time.FixedZone("EDT", -4*3600)) + + got := CombinedFilter("", from, to) + want := `timestamp >= "2026-07-25T00:00:00.25Z" AND timestamp <= "2026-07-25T05:00:00.123456789Z"` + if got != want { + t.Fatalf("CombinedFilter fractional bounds = %q, want %q", got, want) + } +} diff --git a/pkg/gcplog/fetch.go b/pkg/gcplog/fetch.go new file mode 100644 index 0000000..92274c4 --- /dev/null +++ b/pkg/gcplog/fetch.go @@ -0,0 +1,113 @@ +package gcplog + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "cloud.google.com/go/logging" + "cloud.google.com/go/logging/logadmin" + "github.com/go-errors/errors" + "google.golang.org/api/iterator" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" +) + +// adcHint is appended to fetch failures so credential problems point people +// at Application Default Credentials setup. LAPP never manages provider auth. +const adcHint = "if this is a credential problem, run `gcloud auth application-default login`" + +// FetchRequest describes one pull from GCP Cloud Logging. Filter is the user +// supplied Cloud Logging filter; the time range is combined with it when +// querying. Limit caps the number of returned entries. +type FetchRequest struct { + Project string + Filter string + From time.Time + To time.Time + Limit int +} + +// FetchResult carries converted envelope NDJSON lines. Truncated is true +// when the limit cut the result short. +type FetchResult struct { + Lines []string + Truncated bool +} + +// FetchLines pulls matching entries from GCP Cloud Logging using Application +// Default Credentials and converts each one to an envelope NDJSON line. +func FetchLines(ctx context.Context, req FetchRequest) (FetchResult, error) { + client, err := logadmin.NewClient(ctx, req.Project) + if err != nil { + return FetchResult{}, errors.Errorf("create cloud logging client (%s): %w", adcHint, err) + } + defer func() { _ = client.Close() }() + + it := client.Entries(ctx, logadmin.Filter(CombinedFilter(req.Filter, req.From, req.To))) + var result FetchResult + for { + entry, err := it.Next() + if errors.Is(err, iterator.Done) { + break + } + if err != nil { + return FetchResult{}, errors.Errorf("list cloud logging entries (%s): %w", adcHint, err) + } + if req.Limit > 0 && len(result.Lines) >= req.Limit { + result.Truncated = true + break + } + line, err := ConvertEntry(entryFromLogging(entry)) + if err != nil { + return FetchResult{}, err + } + result.Lines = append(result.Lines, line) + } + return result, nil +} + +// CombinedFilter joins the resolved time range with the user supplied filter. +// The time range is a separate concept from the filter; they are only +// combined here at query time. +func CombinedFilter(filter string, from, to time.Time) string { + parts := []string{ + fmt.Sprintf("timestamp >= %q", from.UTC().Format(time.RFC3339Nano)), + fmt.Sprintf("timestamp <= %q", to.UTC().Format(time.RFC3339Nano)), + } + if strings.TrimSpace(filter) != "" { + parts = append(parts, "("+filter+")") + } + return strings.Join(parts, " AND ") +} + +// entryFromLogging maps a client library entry onto the neutral Entry shape +// consumed by ConvertEntry. +func entryFromLogging(e *logging.Entry) Entry { + out := Entry{Timestamp: e.Timestamp} + if e.Severity != logging.Default { + out.Severity = strings.ToUpper(e.Severity.String()) + } + switch payload := e.Payload.(type) { + case nil: + case string: + out.TextPayload = &payload + case proto.Message: + if data, err := protojson.Marshal(payload); err == nil { + out.JSONPayload = data + } else { + text := fmt.Sprint(payload) + out.TextPayload = &text + } + default: + if data, err := json.Marshal(payload); err == nil { + out.JSONPayload = data + } else { + text := fmt.Sprint(payload) + out.TextPayload = &text + } + } + return out +} diff --git a/pkg/gcplog/testdata/convert/cloud-tasks.expected.txt b/pkg/gcplog/testdata/convert/cloud-tasks.expected.txt new file mode 100644 index 0000000..d4232a1 --- /dev/null +++ b/pkg/gcplog/testdata/convert/cloud-tasks.expected.txt @@ -0,0 +1 @@ +{"ts":"2026-07-24T21:20:26.488845Z","severity":"WARNING","payload":{"cloudTasks":{"eta":"1782426638.26684","executionCount":0,"queueDelayMs":2501387518,"queueName":"order-webhook","retryCount":0,"taskName":"1234567890123456789"},"durationMs":704,"level":"warn","message":"TASK POST /tasks/order/webhook 200 in 704ms ","requestId":"00000000-0000-4000-8000-000000000001","spanId":"1111111111111111111"}} diff --git a/pkg/gcplog/testdata/convert/cloud-tasks.input.json b/pkg/gcplog/testdata/convert/cloud-tasks.input.json new file mode 100644 index 0000000..da95a40 --- /dev/null +++ b/pkg/gcplog/testdata/convert/cloud-tasks.input.json @@ -0,0 +1,19 @@ +{ + "timestamp": "2026-07-24T21:20:26.488845Z", + "severity": "WARNING", + "jsonPayload": { + "cloudTasks": { + "eta": "1782426638.26684", + "executionCount": 0, + "queueDelayMs": 2501387518, + "queueName": "order-webhook", + "retryCount": 0, + "taskName": "1234567890123456789" + }, + "durationMs": 704, + "level": "warn", + "message": "TASK POST /tasks/order/webhook 200 in 704ms ", + "requestId": "00000000-0000-4000-8000-000000000001", + "spanId": "1111111111111111111" + } +} diff --git a/pkg/gcplog/testdata/convert/embedded-text.expected.txt b/pkg/gcplog/testdata/convert/embedded-text.expected.txt new file mode 100644 index 0000000..454a2b3 --- /dev/null +++ b/pkg/gcplog/testdata/convert/embedded-text.expected.txt @@ -0,0 +1 @@ +{"ts":"2026-07-24T21:20:26.497706Z","severity":"DEFAULT","payload":{"message":"2026-07-24 21:20:26,496 [INFO] httpx: HTTP Request: PATCH https://api.example.internal/rest/v1/creators?uid=eq.1234567890 \"HTTP/2 200 OK\""}} diff --git a/pkg/gcplog/testdata/convert/embedded-text.input.json b/pkg/gcplog/testdata/convert/embedded-text.input.json new file mode 100644 index 0000000..46a6bc9 --- /dev/null +++ b/pkg/gcplog/testdata/convert/embedded-text.input.json @@ -0,0 +1,7 @@ +{ + "timestamp": "2026-07-24T21:20:26.497706Z", + "severity": "DEFAULT", + "jsonPayload": { + "message": "2026-07-24 21:20:26,496 [INFO] httpx: HTTP Request: PATCH https://api.example.internal/rest/v1/creators?uid=eq.1234567890 \"HTTP/2 200 OK\"" + } +} diff --git a/pkg/gcplog/testdata/convert/empty-text-payload.expected.txt b/pkg/gcplog/testdata/convert/empty-text-payload.expected.txt new file mode 100644 index 0000000..d9d5d61 --- /dev/null +++ b/pkg/gcplog/testdata/convert/empty-text-payload.expected.txt @@ -0,0 +1 @@ +{"ts":"2026-07-01T12:00:04Z","severity":"INFO","payload":{"message":""}} diff --git a/pkg/gcplog/testdata/convert/empty-text-payload.input.json b/pkg/gcplog/testdata/convert/empty-text-payload.input.json new file mode 100644 index 0000000..882c718 --- /dev/null +++ b/pkg/gcplog/testdata/convert/empty-text-payload.input.json @@ -0,0 +1,5 @@ +{ + "timestamp": "2026-07-01T12:00:04Z", + "severity": "INFO", + "textPayload": "" +} diff --git a/pkg/gcplog/testdata/convert/fractional-seconds-offset.expected.txt b/pkg/gcplog/testdata/convert/fractional-seconds-offset.expected.txt new file mode 100644 index 0000000..eae772f --- /dev/null +++ b/pkg/gcplog/testdata/convert/fractional-seconds-offset.expected.txt @@ -0,0 +1 @@ +{"ts":"2026-07-25T14:30:00.25Z","severity":"DEBUG","payload":{"message":"cache warmed"}} diff --git a/pkg/gcplog/testdata/convert/fractional-seconds-offset.input.json b/pkg/gcplog/testdata/convert/fractional-seconds-offset.input.json new file mode 100644 index 0000000..fb12e27 --- /dev/null +++ b/pkg/gcplog/testdata/convert/fractional-seconds-offset.input.json @@ -0,0 +1,5 @@ +{ + "timestamp": "2026-07-25T10:30:00.250-04:00", + "severity": "DEBUG", + "textPayload": "cache warmed" +} diff --git a/pkg/gcplog/testdata/convert/iam-audit.expected.txt b/pkg/gcplog/testdata/convert/iam-audit.expected.txt new file mode 100644 index 0000000..9c39ae6 --- /dev/null +++ b/pkg/gcplog/testdata/convert/iam-audit.expected.txt @@ -0,0 +1 @@ +{"ts":"2026-07-24T21:20:26.489299279Z","severity":"NOTICE","payload":{"serviceName":"iam.googleapis.com","methodName":"iam.serviceAccounts.actAs","resourceName":"projects/-/serviceAccounts/deploy-robot@example-project.iam.gserviceaccount.com","authenticationInfo":{"principalEmail":"deploy-robot@example-project.iam.gserviceaccount.com"},"authorizationInfo":[{"resource":"projects/-/serviceAccounts/deploy-robot@example-project.iam.gserviceaccount.com","permission":"iam.serviceAccounts.actAs","granted":true}],"request":{"@type":"type.googleapis.com/CanActAsServiceAccountRequest","name":"deploy-robot@example-project.iam.gserviceaccount.com","project_number":"123456789012"},"response":{"@type":"type.googleapis.com/CanActAsServiceAccountResponse","success":true}}} diff --git a/pkg/gcplog/testdata/convert/iam-audit.input.json b/pkg/gcplog/testdata/convert/iam-audit.input.json new file mode 100644 index 0000000..709ea62 --- /dev/null +++ b/pkg/gcplog/testdata/convert/iam-audit.input.json @@ -0,0 +1,28 @@ +{ + "timestamp": "2026-07-24T21:20:26.489299279Z", + "severity": "NOTICE", + "jsonPayload": { + "serviceName": "iam.googleapis.com", + "methodName": "iam.serviceAccounts.actAs", + "resourceName": "projects/-/serviceAccounts/deploy-robot@example-project.iam.gserviceaccount.com", + "authenticationInfo": { + "principalEmail": "deploy-robot@example-project.iam.gserviceaccount.com" + }, + "authorizationInfo": [ + { + "resource": "projects/-/serviceAccounts/deploy-robot@example-project.iam.gserviceaccount.com", + "permission": "iam.serviceAccounts.actAs", + "granted": true + } + ], + "request": { + "@type": "type.googleapis.com/CanActAsServiceAccountRequest", + "name": "deploy-robot@example-project.iam.gserviceaccount.com", + "project_number": "123456789012" + }, + "response": { + "@type": "type.googleapis.com/CanActAsServiceAccountResponse", + "success": true + } + } +} diff --git a/pkg/gcplog/testdata/convert/json-payload.expected.txt b/pkg/gcplog/testdata/convert/json-payload.expected.txt new file mode 100644 index 0000000..fac7f19 --- /dev/null +++ b/pkg/gcplog/testdata/convert/json-payload.expected.txt @@ -0,0 +1 @@ +{"ts":"2026-07-01T12:00:00Z","severity":"ERROR","payload":{"message":"payment declined","order_id":1001,"retriable":false}} diff --git a/pkg/gcplog/testdata/convert/json-payload.input.json b/pkg/gcplog/testdata/convert/json-payload.input.json new file mode 100644 index 0000000..6358aec --- /dev/null +++ b/pkg/gcplog/testdata/convert/json-payload.input.json @@ -0,0 +1,9 @@ +{ + "timestamp": "2026-07-01T12:00:00Z", + "severity": "ERROR", + "jsonPayload": { + "message": "payment declined", + "order_id": 1001, + "retriable": false + } +} diff --git a/pkg/gcplog/testdata/convert/missing-severity.expected.txt b/pkg/gcplog/testdata/convert/missing-severity.expected.txt new file mode 100644 index 0000000..0f32606 --- /dev/null +++ b/pkg/gcplog/testdata/convert/missing-severity.expected.txt @@ -0,0 +1 @@ +{"ts":"2026-07-01T12:00:02Z","severity":"DEFAULT","payload":{"message":"started worker pool"}} diff --git a/pkg/gcplog/testdata/convert/missing-severity.input.json b/pkg/gcplog/testdata/convert/missing-severity.input.json new file mode 100644 index 0000000..da29565 --- /dev/null +++ b/pkg/gcplog/testdata/convert/missing-severity.input.json @@ -0,0 +1,4 @@ +{ + "timestamp": "2026-07-01T12:00:02Z", + "textPayload": "started worker pool" +} diff --git a/pkg/gcplog/testdata/convert/no-payload.expected.txt b/pkg/gcplog/testdata/convert/no-payload.expected.txt new file mode 100644 index 0000000..f05ccc8 --- /dev/null +++ b/pkg/gcplog/testdata/convert/no-payload.expected.txt @@ -0,0 +1 @@ +{"ts":"2026-07-01T12:00:03Z","severity":"NOTICE","payload":{}} diff --git a/pkg/gcplog/testdata/convert/no-payload.input.json b/pkg/gcplog/testdata/convert/no-payload.input.json new file mode 100644 index 0000000..f809875 --- /dev/null +++ b/pkg/gcplog/testdata/convert/no-payload.input.json @@ -0,0 +1,4 @@ +{ + "timestamp": "2026-07-01T12:00:03Z", + "severity": "NOTICE" +} diff --git a/pkg/gcplog/testdata/convert/non-utc-offset.expected.txt b/pkg/gcplog/testdata/convert/non-utc-offset.expected.txt new file mode 100644 index 0000000..df6589b --- /dev/null +++ b/pkg/gcplog/testdata/convert/non-utc-offset.expected.txt @@ -0,0 +1 @@ +{"ts":"2026-07-25T02:30:00Z","severity":"INFO","payload":{"message":"user login","user_id":"u-42"}} diff --git a/pkg/gcplog/testdata/convert/non-utc-offset.input.json b/pkg/gcplog/testdata/convert/non-utc-offset.input.json new file mode 100644 index 0000000..cfa6c22 --- /dev/null +++ b/pkg/gcplog/testdata/convert/non-utc-offset.input.json @@ -0,0 +1,8 @@ +{ + "timestamp": "2026-07-25T10:30:00+08:00", + "severity": "INFO", + "jsonPayload": { + "message": "user login", + "user_id": "u-42" + } +} diff --git a/pkg/gcplog/testdata/convert/text-payload.expected.txt b/pkg/gcplog/testdata/convert/text-payload.expected.txt new file mode 100644 index 0000000..4a683e8 --- /dev/null +++ b/pkg/gcplog/testdata/convert/text-payload.expected.txt @@ -0,0 +1 @@ +{"ts":"2026-07-01T12:00:01Z","severity":"WARNING","payload":{"message":"disk usage at 91 percent on node a7"}} diff --git a/pkg/gcplog/testdata/convert/text-payload.input.json b/pkg/gcplog/testdata/convert/text-payload.input.json new file mode 100644 index 0000000..23cbc64 --- /dev/null +++ b/pkg/gcplog/testdata/convert/text-payload.input.json @@ -0,0 +1,5 @@ +{ + "timestamp": "2026-07-01T12:00:01Z", + "severity": "WARNING", + "textPayload": "disk usage at 91 percent on node a7" +} diff --git a/pkg/workspace/import.go b/pkg/workspace/import.go new file mode 100644 index 0000000..91d396f --- /dev/null +++ b/pkg/workspace/import.go @@ -0,0 +1,174 @@ +package workspace + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "strings" + "time" + + "github.com/go-errors/errors" + "github.com/google/uuid" +) + +// ImportRequest is what the fetcher needs to pull entries from a provider. +type ImportRequest struct { + Project string + Filter string + From time.Time + To time.Time + Limit int +} + +// ImportFetchResult carries provider entries already converted to envelope +// NDJSON lines. Truncated is true when the limit cut the result short. +type ImportFetchResult struct { + Lines []string + Truncated bool +} + +// ImportFetcher pulls matching entries from a provider. It is the narrow +// seam between run orchestration and provider network access, so tests can +// inject a fake without touching the network (precedent: DiscoveryConfig.Labeler). +type ImportFetcher func(ctx context.Context, req ImportRequest) (ImportFetchResult, error) + +type ImportConfig struct { + Dir string + RunID string + Provider string + Project string + Filter string + From time.Time + To time.Time + Limit int + Fetcher ImportFetcher +} + +// ImportResult summarizes a completed import run. +type ImportResult struct { + RunID string + EntryCount int + Truncated bool + LogFileName string +} + +// RunImport starts an ImportRun: it records provenance under +// import-runs//record.json, pulls entries through the fetcher, and +// lands them as one NDJSON log file. It never starts discovery (ADR 0005). +// Zero matching entries is a success with no log file written. +func RunImport(ctx context.Context, config ImportConfig) (ImportResult, error) { + if config.Dir == "" { + return ImportResult{}, errors.New("workspace dir is required") + } + if config.Provider == "" { + return ImportResult{}, errors.New("import provider is required") + } + if config.Fetcher == nil { + return ImportResult{}, errors.New("import fetcher is required") + } + runID, err := importRunID(config.RunID) + if err != nil { + return ImportResult{}, err + } + if _, err := os.Stat(ImportRunRecordPath(config.Dir, runID)); err == nil { + return ImportResult{}, errors.Errorf("import run %q already exists", runID) + } else if !os.IsNotExist(err) { + return ImportResult{}, errors.Errorf("check import run %q: %w", runID, err) + } + + record := ImportRunRecord{ + ID: runID, + Provider: config.Provider, + Project: config.Project, + Filter: config.Filter, + From: config.From.UTC(), + To: config.To.UTC(), + Limit: config.Limit, + State: ImportRunStateRunning, + StartedAt: timeNow(), + } + slog.Info("ImportRun started", "run", runID, "provider", config.Provider, "project", config.Project) + if err := WriteImportRunRecord(config.Dir, record); err != nil { + return ImportResult{}, err + } + + fetched, err := config.Fetcher(ctx, ImportRequest{ + Project: config.Project, + Filter: config.Filter, + From: record.From, + To: record.To, + Limit: config.Limit, + }) + if err != nil { + failImportRun(config.Dir, &record, "FETCH_FAILED", err) + return ImportResult{}, err + } + // The limit is enforced here as well as in the fetcher so the cap holds + // regardless of provider behavior. + if config.Limit > 0 && len(fetched.Lines) > config.Limit { + fetched.Lines = fetched.Lines[:config.Limit] + fetched.Truncated = true + } + + if len(fetched.Lines) > 0 { + fileName := config.Provider + "-" + runID + ".ndjson" + logPath := filepath.Join(config.Dir, "logs", fileName) + content := strings.Join(fetched.Lines, "\n") + "\n" + if err := os.WriteFile(logPath, []byte(content), 0o644); err != nil { + err = errors.Errorf("write imported log file: %w", err) + failImportRun(config.Dir, &record, "WRITE_LOG_FAILED", err) + return ImportResult{}, err + } + record.LogFileName = fileName + } + + finishedAt := timeNow() + record.State = ImportRunStateSucceeded + record.FinishedAt = &finishedAt + record.EntryCount = len(fetched.Lines) + record.Truncated = fetched.Truncated + if err := WriteImportRunRecord(config.Dir, record); err != nil { + // Best effort: do not leave the run stuck in RUNNING when the final + // record write fails. A secondary failure here is ignored. + failImportRun(config.Dir, &record, "WRITE_RECORD_FAILED", err) + return ImportResult{}, err + } + + slog.Info( + "ImportRun succeeded", + "run", runID, + "entries", record.EntryCount, + "truncated", record.Truncated, + "file", record.LogFileName, + ) + return ImportResult{ + RunID: runID, + EntryCount: record.EntryCount, + Truncated: record.Truncated, + LogFileName: record.LogFileName, + }, nil +} + +func importRunID(value string) (string, error) { + if value != "" { + return value, nil + } + id, err := uuid.NewV7() + if err != nil { + return "", errors.Errorf("create import run id: %w", err) + } + return id.String(), nil +} + +func failImportRun(workspaceDir string, record *ImportRunRecord, code string, cause error) { + finishedAt := timeNow() + record.State = ImportRunStateFailed + record.FinishedAt = &finishedAt + record.Error = &ImportRunError{ + Code: code, + Message: cause.Error(), + } + slog.Error("ImportRun failed", "run", record.ID, "code", code, "error", cause) + _ = WriteImportRunRecord(workspaceDir, *record) +} diff --git a/pkg/workspace/import_test.go b/pkg/workspace/import_test.go new file mode 100644 index 0000000..a7c9a15 --- /dev/null +++ b/pkg/workspace/import_test.go @@ -0,0 +1,244 @@ +package workspace + +import ( + "context" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/go-errors/errors" +) + +func importTestConfig(dir string, fetcher ImportFetcher) ImportConfig { + return ImportConfig{ + Dir: dir, + RunID: "01900000-0000-7000-8000-00000000b001", + Provider: "gcp", + Project: "acme-prod", + Filter: `severity>=ERROR`, + From: time.Date(2026, 7, 25, 0, 0, 0, 0, time.UTC), + To: time.Date(2026, 7, 25, 1, 0, 0, 0, time.UTC), + Limit: 100, + Fetcher: fetcher, + } +} + +func TestRunImportSuccessWritesFileAndRecord(t *testing.T) { + dir := t.TempDir() + mustMkdir(t, filepath.Join(dir, "logs")) + lines := []string{ + `{"ts":"2026-07-25T00:00:01Z","severity":"ERROR","payload":{"message":"a"}}`, + `{"ts":"2026-07-25T00:00:02Z","severity":"ERROR","payload":{"message":"b"}}`, + } + var gotReq ImportRequest + fetcher := func(_ context.Context, req ImportRequest) (ImportFetchResult, error) { + gotReq = req + return ImportFetchResult{Lines: lines}, nil + } + + result, err := RunImport(context.Background(), importTestConfig(dir, fetcher)) + if err != nil { + t.Fatalf("RunImport: %v", err) + } + if result.EntryCount != 2 || result.Truncated { + t.Fatalf("unexpected result: %+v", result) + } + wantFile := "gcp-01900000-0000-7000-8000-00000000b001.ndjson" + if result.LogFileName != wantFile { + t.Fatalf("log file name = %q, want %q", result.LogFileName, wantFile) + } + if gotReq.Project != "acme-prod" || gotReq.Filter != `severity>=ERROR` || gotReq.Limit != 100 { + t.Fatalf("unexpected fetch request: %+v", gotReq) + } + + content := mustRead(t, filepath.Join(dir, "logs", wantFile)) + if content != lines[0]+"\n"+lines[1]+"\n" { + t.Fatalf("unexpected log file content:\n%s", content) + } + + record, err := ReadImportRunRecord(dir, result.RunID) + if err != nil { + t.Fatalf("ReadImportRunRecord: %v", err) + } + if record.State != ImportRunStateSucceeded { + t.Fatalf("record state = %s, want SUCCEEDED", record.State) + } + if record.EntryCount != 2 || record.Truncated || record.LogFileName != wantFile { + t.Fatalf("unexpected record: %+v", record) + } + if record.Provider != "gcp" || record.Project != "acme-prod" || record.Limit != 100 { + t.Fatalf("unexpected record provenance: %+v", record) + } + if record.FinishedAt == nil || record.Error != nil { + t.Fatalf("expected finished record without error: %+v", record) + } +} + +func TestRunImportZeroEntriesSucceedsWithoutFile(t *testing.T) { + dir := t.TempDir() + mustMkdir(t, filepath.Join(dir, "logs")) + fetcher := func(_ context.Context, _ ImportRequest) (ImportFetchResult, error) { + return ImportFetchResult{}, nil + } + + result, err := RunImport(context.Background(), importTestConfig(dir, fetcher)) + if err != nil { + t.Fatalf("RunImport: %v", err) + } + if result.EntryCount != 0 || result.LogFileName != "" { + t.Fatalf("unexpected result: %+v", result) + } + + logEntries, err := os.ReadDir(filepath.Join(dir, "logs")) + if err != nil { + t.Fatalf("read logs dir: %v", err) + } + if len(logEntries) != 0 { + t.Fatalf("expected no log files, found %d", len(logEntries)) + } + + record, err := ReadImportRunRecord(dir, result.RunID) + if err != nil { + t.Fatalf("ReadImportRunRecord: %v", err) + } + if record.State != ImportRunStateSucceeded || record.EntryCount != 0 || record.LogFileName != "" { + t.Fatalf("unexpected record: %+v", record) + } +} + +func TestRunImportTruncationIsRecorded(t *testing.T) { + dir := t.TempDir() + mustMkdir(t, filepath.Join(dir, "logs")) + fetcher := func(_ context.Context, _ ImportRequest) (ImportFetchResult, error) { + return ImportFetchResult{ + Lines: []string{`{"ts":"2026-07-25T00:00:01Z","severity":"ERROR","payload":{"message":"a"}}`}, + Truncated: true, + }, nil + } + + result, err := RunImport(context.Background(), importTestConfig(dir, fetcher)) + if err != nil { + t.Fatalf("RunImport: %v", err) + } + if !result.Truncated { + t.Fatalf("expected truncated result: %+v", result) + } + + record, err := ReadImportRunRecord(dir, result.RunID) + if err != nil { + t.Fatalf("ReadImportRunRecord: %v", err) + } + if !record.Truncated || record.State != ImportRunStateSucceeded { + t.Fatalf("unexpected record: %+v", record) + } +} + +func TestRunImportFetchFailureWritesFailedRecord(t *testing.T) { + dir := t.TempDir() + mustMkdir(t, filepath.Join(dir, "logs")) + fetcher := func(_ context.Context, _ ImportRequest) (ImportFetchResult, error) { + return ImportFetchResult{}, errors.New("could not find default credentials") + } + + config := importTestConfig(dir, fetcher) + if _, err := RunImport(context.Background(), config); err == nil { + t.Fatal("expected fetch error") + } + + record, err := ReadImportRunRecord(dir, config.RunID) + if err != nil { + t.Fatalf("ReadImportRunRecord: %v", err) + } + if record.State != ImportRunStateFailed { + t.Fatalf("record state = %s, want FAILED", record.State) + } + if record.Error == nil || record.Error.Code != "FETCH_FAILED" || record.Error.Message == "" { + t.Fatalf("unexpected record error: %+v", record.Error) + } + if record.FinishedAt == nil { + t.Fatal("expected finished timestamp on failed record") + } + + logEntries, err := os.ReadDir(filepath.Join(dir, "logs")) + if err != nil { + t.Fatalf("read logs dir: %v", err) + } + if len(logEntries) != 0 { + t.Fatalf("expected no log files, found %d", len(logEntries)) + } +} + +func TestRunImportCapsLinesAtLimit(t *testing.T) { + dir := t.TempDir() + mustMkdir(t, filepath.Join(dir, "logs")) + fetcher := func(_ context.Context, _ ImportRequest) (ImportFetchResult, error) { + return ImportFetchResult{ + Lines: []string{ + `{"ts":"2026-07-25T00:00:01Z","severity":"ERROR","payload":{"message":"a"}}`, + `{"ts":"2026-07-25T00:00:02Z","severity":"ERROR","payload":{"message":"b"}}`, + `{"ts":"2026-07-25T00:00:03Z","severity":"ERROR","payload":{"message":"c"}}`, + }, + }, nil + } + + config := importTestConfig(dir, fetcher) + config.Limit = 2 + result, err := RunImport(context.Background(), config) + if err != nil { + t.Fatalf("RunImport: %v", err) + } + if result.EntryCount != 2 || !result.Truncated { + t.Fatalf("expected capped truncated result, got %+v", result) + } + + content := mustRead(t, filepath.Join(dir, "logs", result.LogFileName)) + lines := strings.Split(strings.TrimSuffix(content, "\n"), "\n") + if len(lines) != 2 { + t.Fatalf("expected exactly 2 lines written, got %d:\n%s", len(lines), content) + } +} + +func TestRunImportRejectsDuplicateRunID(t *testing.T) { + dir := t.TempDir() + mustMkdir(t, filepath.Join(dir, "logs")) + fetcher := func(_ context.Context, _ ImportRequest) (ImportFetchResult, error) { + return ImportFetchResult{ + Lines: []string{`{"ts":"2026-07-25T00:00:01Z","severity":"ERROR","payload":{"message":"a"}}`}, + }, nil + } + + config := importTestConfig(dir, fetcher) + existing := ImportRunRecord{ + ID: config.RunID, + Provider: "gcp", + Project: "acme-prod", + State: ImportRunStateSucceeded, + StartedAt: time.Date(2026, 7, 24, 0, 0, 0, 0, time.UTC), + } + if err := WriteImportRunRecord(dir, existing); err != nil { + t.Fatalf("WriteImportRunRecord: %v", err) + } + + if _, err := RunImport(context.Background(), config); err == nil { + t.Fatal("expected error for duplicate run id") + } + + record, err := ReadImportRunRecord(dir, config.RunID) + if err != nil { + t.Fatalf("ReadImportRunRecord: %v", err) + } + if !reflect.DeepEqual(record, existing) { + t.Fatalf("existing record was modified\ngot: %+v\nwant: %+v", record, existing) + } + + logEntries, err := os.ReadDir(filepath.Join(dir, "logs")) + if err != nil { + t.Fatalf("read logs dir: %v", err) + } + if len(logEntries) != 0 { + t.Fatalf("expected no log files, found %d", len(logEntries)) + } +} diff --git a/pkg/workspace/importrun.go b/pkg/workspace/importrun.go new file mode 100644 index 0000000..a8531e2 --- /dev/null +++ b/pkg/workspace/importrun.go @@ -0,0 +1,124 @@ +package workspace + +import ( + "encoding/json" + "os" + "path/filepath" + "time" + + "github.com/go-errors/errors" +) + +const ImportRunsDirName = "import-runs" + +type ImportRunState string + +const ( + ImportRunStateQueued ImportRunState = "QUEUED" + ImportRunStateRunning ImportRunState = "RUNNING" + ImportRunStateSucceeded ImportRunState = "SUCCEEDED" + ImportRunStateFailed ImportRunState = "FAILED" +) + +// ImportRunRecord is the persisted provenance for one provider import. +// These records are the only store of import history (ADR 0004). Provider +// metadata such as labels, trace, and resource belongs here, never in the +// imported log lines. +type ImportRunRecord struct { + ID string `json:"id"` + Provider string `json:"provider"` + Project string `json:"project"` + Filter string `json:"filter,omitempty"` + From time.Time `json:"from"` + To time.Time `json:"to"` + Limit int `json:"limit"` + State ImportRunState `json:"state"` + EntryCount int `json:"entry_count"` + Truncated bool `json:"truncated"` + LogFileName string `json:"log_file_name,omitempty"` + StartedAt time.Time `json:"started_at"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + Error *ImportRunError `json:"error,omitempty"` +} + +type ImportRunError struct { + Code string `json:"code"` + Message string `json:"message"` +} + +func ImportRunsDir(workspaceDir string) string { + return filepath.Join(workspaceDir, ImportRunsDirName) +} + +func ImportRunDir(workspaceDir, runID string) string { + return filepath.Join(ImportRunsDir(workspaceDir), runID) +} + +func ImportRunRecordPath(workspaceDir, runID string) string { + return filepath.Join(ImportRunDir(workspaceDir, runID), "record.json") +} + +func WriteImportRunRecord(workspaceDir string, record ImportRunRecord) error { + if record.ID == "" { + return errors.New("import run id is required") + } + runDir := ImportRunDir(workspaceDir, record.ID) + if err := os.MkdirAll(runDir, 0o755); err != nil { + return errors.Errorf("create import run dir: %w", err) + } + data, err := json.MarshalIndent(record, "", " ") + if err != nil { + return errors.Errorf("marshal import run: %w", err) + } + tmp, err := os.CreateTemp(runDir, "record-*.json.tmp") + if err != nil { + return errors.Errorf("create temp import run: %w", err) + } + tmpPath := tmp.Name() + defer func() { _ = os.Remove(tmpPath) }() + if _, err := tmp.Write(append(data, '\n')); err != nil { + _ = tmp.Close() + return errors.Errorf("write temp import run: %w", err) + } + if err := tmp.Close(); err != nil { + return errors.Errorf("close temp import run: %w", err) + } + if err := os.Rename(tmpPath, ImportRunRecordPath(workspaceDir, record.ID)); err != nil { + return errors.Errorf("write import run: %w", err) + } + return nil +} + +func ReadImportRunRecord(workspaceDir, runID string) (ImportRunRecord, error) { + data, err := os.ReadFile(ImportRunRecordPath(workspaceDir, runID)) + if err != nil { + return ImportRunRecord{}, errors.Errorf("read import run: %w", err) + } + var record ImportRunRecord + if err := json.Unmarshal(data, &record); err != nil { + return ImportRunRecord{}, errors.Errorf("decode import run: %w", err) + } + return record, nil +} + +func ListImportRunRecords(workspaceDir string) ([]ImportRunRecord, error) { + entries, err := os.ReadDir(ImportRunsDir(workspaceDir)) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, errors.Errorf("read import runs dir: %w", err) + } + records := make([]ImportRunRecord, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + record, err := ReadImportRunRecord(workspaceDir, entry.Name()) + if err != nil { + return nil, err + } + records = append(records, record) + } + return records, nil +} diff --git a/pkg/workspace/importrun_test.go b/pkg/workspace/importrun_test.go new file mode 100644 index 0000000..ac77473 --- /dev/null +++ b/pkg/workspace/importrun_test.go @@ -0,0 +1,58 @@ +package workspace + +import ( + "reflect" + "testing" + "time" +) + +func TestImportRunRecordRoundTrip(t *testing.T) { + dir := t.TempDir() + finishedAt := time.Date(2026, 7, 25, 1, 2, 3, 0, time.UTC) + record := ImportRunRecord{ + ID: "01900000-0000-7000-8000-00000000a001", + Provider: "gcp", + Project: "acme-prod", + Filter: `resource.type="cloud_run_revision"`, + From: time.Date(2026, 7, 25, 0, 0, 0, 0, time.UTC), + To: time.Date(2026, 7, 25, 1, 0, 0, 0, time.UTC), + Limit: 100000, + State: ImportRunStateSucceeded, + EntryCount: 42, + Truncated: true, + LogFileName: "gcp-01900000-0000-7000-8000-00000000a001.ndjson", + StartedAt: time.Date(2026, 7, 25, 1, 0, 5, 0, time.UTC), + FinishedAt: &finishedAt, + Error: &ImportRunError{ + Code: "FETCH_FAILED", + Message: "boom", + }, + } + + if err := WriteImportRunRecord(dir, record); err != nil { + t.Fatalf("WriteImportRunRecord: %v", err) + } + got, err := ReadImportRunRecord(dir, record.ID) + if err != nil { + t.Fatalf("ReadImportRunRecord: %v", err) + } + if !reflect.DeepEqual(got, record) { + t.Fatalf("round trip mismatch\ngot: %+v\nwant: %+v", got, record) + } +} + +func TestWriteImportRunRecordRequiresID(t *testing.T) { + if err := WriteImportRunRecord(t.TempDir(), ImportRunRecord{}); err == nil { + t.Fatal("expected error for missing run id") + } +} + +func TestListImportRunRecordsMissingDirIsEmpty(t *testing.T) { + records, err := ListImportRunRecords(t.TempDir()) + if err != nil { + t.Fatalf("ListImportRunRecords: %v", err) + } + if len(records) != 0 { + t.Fatalf("expected no records, got %d", len(records)) + } +}