diff --git a/CONTEXT.md b/CONTEXT.md index 43da09e..34b85f9 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -27,6 +27,6 @@ _Avoid_: Sync, connection, integration, LogImport An external logging service LAPP can import from, such as GCP Cloud Logging or Vercel. LAPP uses credentials already present on the machine and never manages provider authentication itself. _Avoid_: Source, backend -**Projection**: -The text line derived from a structured log entry for pattern discovery. Discovery reads the projection; investigation material keeps the full structured entry. -_Avoid_: Flattening, rendering +**Extraction**: +The text line pulled out of a structured log entry for pattern discovery. Discovery reads the extraction; investigation material keeps the full structured entry. +_Avoid_: Projection, flattening, rendering diff --git a/docs/adr/0006-ndjson-log-files-with-message-projection.md b/docs/adr/0006-ndjson-log-files-with-message-extraction.md similarity index 69% rename from docs/adr/0006-ndjson-log-files-with-message-projection.md rename to docs/adr/0006-ndjson-log-files-with-message-extraction.md index 03d44b3..e06955c 100644 --- a/docs/adr/0006-ndjson-log-files-with-message-projection.md +++ b/docs/adr/0006-ndjson-log-files-with-message-extraction.md @@ -1,16 +1,16 @@ -# Structured logs are stored as NDJSON and discovered via message projection +# Structured logs are stored as NDJSON and discovered via message extraction Log files can be NDJSON (one JSON entry per line) as well as plain text, detected per file by the pipeline — the capability belongs to the pipeline, not to any import provider, so hand-uploaded NDJSON files get the same treatment as imported ones. Imported GCP entries land in a fixed envelope: `{"ts": ..., "severity": ..., "payload": {...}}` with the jsonPayload nested untouched (textPayload becomes `payload.message`); provider metadata such as labels, trace, and resource stays in the ImportRun record, not in the log lines. -For pattern discovery, JSON entries are projected to a text line rather than fed to Drain whole: the first string field among `payload.message`, `msg`, `log`, `error` becomes the projection (` `); if none exists, the compact payload JSON is the fallback. Timestamps are excluded from the projection to keep Drain templates clean. +For pattern discovery, a text line is extracted from each JSON entry rather than feeding Drain the whole entry: the first string field among `payload.message`, `msg`, `log`, `error` becomes the extraction (` `); if none exists, the compact payload JSON is the fallback. Timestamps are excluded from the extraction to keep Drain templates clean. **Considered Options** - Flatten everything to text at import time (structure lost) - NDJSON storage, discovery on compact-JSON lines (structure kept, dirty templates) -- NDJSON storage with message projection (chosen) +- NDJSON storage with message extraction (chosen) - JSON-native pattern discovery by structure/key-set (a second discovery engine — deferred, not rejected) **Consequences** -The envelope and the projection rule are part of the workspace file contract (ADR 0001). Payload fields are never flattened to the top level, so user fields named `severity` or `ts` cannot collide with the envelope. Analysis agents can query structure with jq-style tools instead of grepping flattened text. If real usage shows dirty templates for payloads without a message-like field, the projection rule is the extension point. +The envelope and the extraction rule are part of the workspace file contract (ADR 0001). Payload fields are never flattened to the top level, so user fields named `severity` or `ts` cannot collide with the envelope. Analysis agents can query structure with jq-style tools instead of grepping flattened text. If real usage shows dirty templates for payloads without a message-like field, the extraction rule is the extension point. diff --git a/pkg/ndjson/detect.go b/pkg/ndjson/detect.go new file mode 100644 index 0000000..630c0fb --- /dev/null +++ b/pkg/ndjson/detect.go @@ -0,0 +1,48 @@ +// Package ndjson classifies log files as NDJSON and extracts the text lines +// fed to pattern mining from structured entries, per ADR 0006. +package ndjson + +import ( + "encoding/json" + "strings" +) + +// Format classifies the on-disk format of a log file. +type Format string + +const ( + // FormatText marks a file for the plain text pipeline. + FormatText Format = "text" + // FormatNDJSON marks a file whose lines are JSON objects, one per line. + FormatNDJSON Format = "ndjson" +) + +// DetectFormat classifies a file's lines. A file is NDJSON only when it has +// at least one non-empty line and every non-empty line parses as a JSON +// object. Files mixing text with JSON lines stay on the text path, so the +// extraction never has to guess on a half-structured file. +func DetectFormat(lines []string) Format { + sampled := 0 + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + if !isJSONObject(trimmed) { + return FormatText + } + sampled++ + } + if sampled == 0 { + return FormatText + } + return FormatNDJSON +} + +func isJSONObject(trimmed string) bool { + if !strings.HasPrefix(trimmed, "{") { + return false + } + var obj map[string]any + return json.Unmarshal([]byte(trimmed), &obj) == nil +} diff --git a/pkg/ndjson/detect_test.go b/pkg/ndjson/detect_test.go new file mode 100644 index 0000000..964a33a --- /dev/null +++ b/pkg/ndjson/detect_test.go @@ -0,0 +1,60 @@ +package ndjson + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// TestDetectFormatFixtures classifies every fixture under testdata/detect/. +// The expected format is encoded in the file name as .expect-.log, +// so adding a case only means adding one fixture file. +func TestDetectFormatFixtures(t *testing.T) { + paths, err := filepath.Glob(filepath.Join("testdata", "detect", "*.log")) + if err != nil { + t.Fatalf("glob detect fixtures: %v", err) + } + if len(paths) == 0 { + t.Fatal("no detect fixtures found under testdata/detect") + } + + for _, path := range paths { + name, want := parseDetectFixtureName(t, path) + t.Run(name, func(t *testing.T) { + got := DetectFormat(readFixtureLines(t, path)) + if got != want { + t.Fatalf("DetectFormat(%s) = %q, want %q", path, got, want) + } + }) + } +} + +func TestDetectFormatEmptyInput(t *testing.T) { + if got := DetectFormat(nil); got != FormatText { + t.Fatalf("DetectFormat(nil) = %q, want %q", got, FormatText) + } +} + +func parseDetectFixtureName(t *testing.T, path string) (string, Format) { + t.Helper() + base := strings.TrimSuffix(filepath.Base(path), ".log") + name, expected, ok := strings.Cut(base, ".expect-") + if !ok { + t.Fatalf("detect fixture %s must be named .expect-.log", path) + } + format := Format(expected) + if format != FormatText && format != FormatNDJSON { + t.Fatalf("detect fixture %s has unknown expected format %q", path, expected) + } + return name, format +} + +func readFixtureLines(t *testing.T, path string) []string { + t.Helper() + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read fixture %s: %v", path, err) + } + return strings.Split(strings.TrimSuffix(string(content), "\n"), "\n") +} diff --git a/pkg/ndjson/extract.go b/pkg/ndjson/extract.go new file mode 100644 index 0000000..bdd6fa4 --- /dev/null +++ b/pkg/ndjson/extract.go @@ -0,0 +1,80 @@ +package ndjson + +import ( + "bytes" + "encoding/json" + "strings" +) + +// messageFields are checked in order; the first string value wins. +var messageFields = []string{"message", "msg", "log", "error"} + +// severityFields are checked in order; the first string value wins. +// Numeric levels are ignored in v1. +var severityFields = []string{"severity", "level"} + +// Extract converts one NDJSON line into the text line fed to pattern mining. +// Envelope entries {"ts": ..., "severity": ..., "payload": {...}} extract +// from the payload; any other object is the payload itself. The extracted line is +// " ", just "" when no severity-like string field +// exists, or the compact payload JSON when no message-like string field +// exists. A line that does not parse as a JSON object is returned unchanged. +func Extract(line string) string { + var entry map[string]any + if err := json.Unmarshal([]byte(line), &entry); err != nil || entry == nil { + return line + } + + payload, severity := splitEnvelope(entry) + if severity == "" { + severity = firstStringField(payload, severityFields) + } + + message := firstStringField(payload, messageFields) + if message == "" { + return compactJSON(payload) + } + if severity == "" { + return message + } + return severity + " " + message +} + +// splitEnvelope returns the payload object and the envelope severity when the +// entry matches the fixed envelope shape, otherwise the entry itself. The +// envelope is the importer's fixed contract (ADR 0006): ts, a string severity, +// and a payload object must all be present; anything less is an arbitrary +// user shape and is extracted as a whole. +func splitEnvelope(entry map[string]any) (payload map[string]any, severity string) { + nested, ok := entry["payload"].(map[string]any) + if !ok { + return entry, "" + } + if _, hasTS := entry["ts"]; !hasTS { + return entry, "" + } + envelopeSeverity, ok := entry["severity"].(string) + if !ok || envelopeSeverity == "" { + return entry, "" + } + return nested, envelopeSeverity +} + +func firstStringField(obj map[string]any, fields []string) string { + for _, field := range fields { + if value, ok := obj[field].(string); ok && value != "" { + return value + } + } + return "" +} + +func compactJSON(payload map[string]any) string { + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(payload); err != nil { + return "" + } + return strings.TrimSuffix(buf.String(), "\n") +} diff --git a/pkg/ndjson/extract_test.go b/pkg/ndjson/extract_test.go new file mode 100644 index 0000000..721e27c --- /dev/null +++ b/pkg/ndjson/extract_test.go @@ -0,0 +1,52 @@ +package ndjson + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// TestExtractFixtures extracts every line of every input fixture under +// testdata/extract/ and compares the result against its committed expected +// output. Each .input.ndjson pairs with .expected.txt, so adding +// a case only means adding one fixture pair. +func TestExtractFixtures(t *testing.T) { + inputs, err := filepath.Glob(filepath.Join("testdata", "extract", "*.input.ndjson")) + if err != nil { + t.Fatalf("glob extraction fixtures: %v", err) + } + if len(inputs) == 0 { + t.Fatal("no extraction fixtures found under testdata/extract") + } + + for _, inputPath := range inputs { + name := strings.TrimSuffix(filepath.Base(inputPath), ".input.ndjson") + expectedPath := filepath.Join("testdata", "extract", name+".expected.txt") + t.Run(name, func(t *testing.T) { + var extracted []string + for _, line := range readFixtureLines(t, inputPath) { + if strings.TrimSpace(line) == "" { + continue + } + extracted = append(extracted, Extract(line)) + } + got := strings.Join(extracted, "\n") + "\n" + + expected, err := os.ReadFile(expectedPath) + if err != nil { + t.Fatalf("read expected output %s: %v", expectedPath, err) + } + if got != string(expected) { + t.Fatalf("extraction mismatch for %s\ngot:\n%swant:\n%s", inputPath, got, expected) + } + }) + } +} + +func TestExtractNonJSONLineIsReturnedUnchanged(t *testing.T) { + line := "2026-06-06 10:00:00 INFO plain text line" + if got := Extract(line); got != line { + t.Fatalf("Extract(%q) = %q, want unchanged", line, got) + } +} diff --git a/pkg/ndjson/testdata/detect/blank-lines-only.expect-text.log b/pkg/ndjson/testdata/detect/blank-lines-only.expect-text.log new file mode 100644 index 0000000..e69de29 diff --git a/pkg/ndjson/testdata/detect/json-arrays.expect-text.log b/pkg/ndjson/testdata/detect/json-arrays.expect-text.log new file mode 100644 index 0000000..57e30db --- /dev/null +++ b/pkg/ndjson/testdata/detect/json-arrays.expect-text.log @@ -0,0 +1,2 @@ +[1,2,3] +["a","b"] diff --git a/pkg/ndjson/testdata/detect/mixed.expect-text.log b/pkg/ndjson/testdata/detect/mixed.expect-text.log new file mode 100644 index 0000000..5f930ce --- /dev/null +++ b/pkg/ndjson/testdata/detect/mixed.expect-text.log @@ -0,0 +1,4 @@ +{"ts":"2026-06-06T10:00:00Z","severity":"ERROR","payload":{"message":"db timeout user=42"}} +{"level":"warn","msg":"queue depth 100"} +2026-06-06 10:00:02 ERROR db timeout user=43 +{"event":"gc","duration_ms":12} diff --git a/pkg/ndjson/testdata/detect/plain-text.expect-text.log b/pkg/ndjson/testdata/detect/plain-text.expect-text.log new file mode 100644 index 0000000..eb7a778 --- /dev/null +++ b/pkg/ndjson/testdata/detect/plain-text.expect-text.log @@ -0,0 +1,3 @@ +2026-06-06 10:00:00 INFO server started port=8080 +2026-06-06 10:00:01 ERROR db timeout user=42 +2026-06-06 10:00:02 ERROR db timeout user=43 diff --git a/pkg/ndjson/testdata/detect/pure-ndjson.expect-ndjson.log b/pkg/ndjson/testdata/detect/pure-ndjson.expect-ndjson.log new file mode 100644 index 0000000..b4476ff --- /dev/null +++ b/pkg/ndjson/testdata/detect/pure-ndjson.expect-ndjson.log @@ -0,0 +1,5 @@ +{"ts":"2026-06-06T10:00:00Z","severity":"ERROR","payload":{"message":"db timeout user=42"}} +{"level":"warn","msg":"queue depth 100"} + +{"event":"gc","duration_ms":12} +{"ts":"2026-06-06T10:00:03Z","severity":"INFO","payload":{"message":"server started port=8080"}} diff --git a/pkg/ndjson/testdata/extract/arbitrary-shape.expected.txt b/pkg/ndjson/testdata/extract/arbitrary-shape.expected.txt new file mode 100644 index 0000000..1be2c14 --- /dev/null +++ b/pkg/ndjson/testdata/extract/arbitrary-shape.expected.txt @@ -0,0 +1,4 @@ +WARN disk usage high volume=/var +connection reset peer=10.0.0.5 +context deadline exceeded +primary text diff --git a/pkg/ndjson/testdata/extract/arbitrary-shape.input.ndjson b/pkg/ndjson/testdata/extract/arbitrary-shape.input.ndjson new file mode 100644 index 0000000..aff73dd --- /dev/null +++ b/pkg/ndjson/testdata/extract/arbitrary-shape.input.ndjson @@ -0,0 +1,4 @@ +{"severity":"WARN","message":"disk usage high volume=/var"} +{"log":"connection reset peer=10.0.0.5"} +{"error":"context deadline exceeded","op":"fetch"} +{"message":"primary text","msg":"secondary text"} diff --git a/pkg/ndjson/testdata/extract/envelope.expected.txt b/pkg/ndjson/testdata/extract/envelope.expected.txt new file mode 100644 index 0000000..1e28b54 --- /dev/null +++ b/pkg/ndjson/testdata/extract/envelope.expected.txt @@ -0,0 +1,3 @@ +ERROR db timeout user=42 +INFO server started port=8080 +{"payload":{"message":"heartbeat ok"},"ts":"2026-06-06T10:00:02Z"} diff --git a/pkg/ndjson/testdata/extract/envelope.input.ndjson b/pkg/ndjson/testdata/extract/envelope.input.ndjson new file mode 100644 index 0000000..8dcb081 --- /dev/null +++ b/pkg/ndjson/testdata/extract/envelope.input.ndjson @@ -0,0 +1,3 @@ +{"ts":"2026-06-06T10:00:00Z","severity":"ERROR","payload":{"message":"db timeout user=42"}} +{"ts":"2026-06-06T10:00:01Z","severity":"INFO","payload":{"msg":"server started port=8080"}} +{"ts":"2026-06-06T10:00:02Z","payload":{"message":"heartbeat ok"}} diff --git a/pkg/ndjson/testdata/extract/no-message-fallback.expected.txt b/pkg/ndjson/testdata/extract/no-message-fallback.expected.txt new file mode 100644 index 0000000..1b0a172 --- /dev/null +++ b/pkg/ndjson/testdata/extract/no-message-fallback.expected.txt @@ -0,0 +1,2 @@ +{"duration_ms":12,"event":"gc"} +{"event":"cache_evict","keys":120} diff --git a/pkg/ndjson/testdata/extract/no-message-fallback.input.ndjson b/pkg/ndjson/testdata/extract/no-message-fallback.input.ndjson new file mode 100644 index 0000000..2811209 --- /dev/null +++ b/pkg/ndjson/testdata/extract/no-message-fallback.input.ndjson @@ -0,0 +1,2 @@ +{"event":"gc","duration_ms":12} +{"ts":"2026-06-06T10:00:05Z","severity":"DEBUG","payload":{"event":"cache_evict","keys":120}} diff --git a/pkg/ndjson/testdata/extract/numeric-level-ignored.expected.txt b/pkg/ndjson/testdata/extract/numeric-level-ignored.expected.txt new file mode 100644 index 0000000..255b7fb --- /dev/null +++ b/pkg/ndjson/testdata/extract/numeric-level-ignored.expected.txt @@ -0,0 +1,2 @@ +request done status=200 +upstream unavailable diff --git a/pkg/ndjson/testdata/extract/numeric-level-ignored.input.ndjson b/pkg/ndjson/testdata/extract/numeric-level-ignored.input.ndjson new file mode 100644 index 0000000..2b05d51 --- /dev/null +++ b/pkg/ndjson/testdata/extract/numeric-level-ignored.input.ndjson @@ -0,0 +1,2 @@ +{"level":30,"msg":"request done status=200"} +{"level":50,"error":"upstream unavailable"} diff --git a/pkg/ndjson/testdata/extract/severity-via-level.expected.txt b/pkg/ndjson/testdata/extract/severity-via-level.expected.txt new file mode 100644 index 0000000..a701c83 --- /dev/null +++ b/pkg/ndjson/testdata/extract/severity-via-level.expected.txt @@ -0,0 +1,2 @@ +warn queue depth 100 +info user login user=alice diff --git a/pkg/ndjson/testdata/extract/severity-via-level.input.ndjson b/pkg/ndjson/testdata/extract/severity-via-level.input.ndjson new file mode 100644 index 0000000..f26a5eb --- /dev/null +++ b/pkg/ndjson/testdata/extract/severity-via-level.input.ndjson @@ -0,0 +1,2 @@ +{"level":"warn","msg":"queue depth 100"} +{"level":"info","message":"user login user=alice"} diff --git a/pkg/workspace/builder.go b/pkg/workspace/builder.go index f52e276..d313fcf 100644 --- a/pkg/workspace/builder.go +++ b/pkg/workspace/builder.go @@ -107,7 +107,7 @@ func (b *Builder) computePatterns() { } matches := make([]lineWithTemplate, 0, len(b.tagged)) for _, tl := range b.tagged { - t, ok := pattern.MatchTemplate(tl.Content, b.templates) + t, ok := pattern.MatchTemplate(tl.DrainLine(), b.templates) id := "" if ok { id = t.ID.String() diff --git a/pkg/workspace/pipeline.go b/pkg/workspace/pipeline.go index 3dc8429..3663c73 100644 --- a/pkg/workspace/pipeline.go +++ b/pkg/workspace/pipeline.go @@ -6,12 +6,14 @@ import ( "os" "path/filepath" "sort" + "strings" "sync" "time" "github.com/go-errors/errors" "github.com/google/uuid" "github.com/strrl/lapp/pkg/multiline" + "github.com/strrl/lapp/pkg/ndjson" "github.com/strrl/lapp/pkg/pattern" "github.com/strrl/lapp/pkg/semantic" "go.opentelemetry.io/otel" @@ -298,24 +300,57 @@ func mergeAllLogs(ctx context.Context, dir string) (tagged []TaggedLine, content var allTagged []TaggedLine var allContent []string for _, fileName := range fileNames { - lines := allLogs[fileName] - detector, err := multiline.NewDetector(multiline.DetectorConfig{}) + fileTagged, err := tagFileLines(ctx, fileName, allLogs[fileName]) if err != nil { - return nil, nil, 0, errors.Errorf("multiline detector: %w", err) + return nil, nil, 0, err } - merged := multiline.MergeSlice(ctx, lines, detector) - for _, m := range merged { - allTagged = append(allTagged, TaggedLine{ - Content: m.Content, - FileName: fileName, - LineNum: m.StartLine, - }) - allContent = append(allContent, m.Content) + for _, tl := range fileTagged { + allTagged = append(allTagged, tl) + allContent = append(allContent, tl.DrainLine()) } } return allTagged, allContent, len(allLogs), nil } +// tagFileLines converts one log file into tagged entries. NDJSON files keep +// the raw JSON line as Content and carry an extracted text line for pattern +// mining; plain text files go through multiline merging unchanged. +func tagFileLines(ctx context.Context, fileName string, lines []string) ([]TaggedLine, error) { + if ndjson.DetectFormat(lines) == ndjson.FormatNDJSON { + return tagNDJSONLines(fileName, lines), nil + } + detector, err := multiline.NewDetector(multiline.DetectorConfig{}) + if err != nil { + return nil, errors.Errorf("multiline detector: %w", err) + } + merged := multiline.MergeSlice(ctx, lines, detector) + tagged := make([]TaggedLine, 0, len(merged)) + for _, m := range merged { + tagged = append(tagged, TaggedLine{ + Content: m.Content, + FileName: fileName, + LineNum: m.StartLine, + }) + } + return tagged, nil +} + +func tagNDJSONLines(fileName string, lines []string) []TaggedLine { + var tagged []TaggedLine + for i, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + tagged = append(tagged, TaggedLine{ + Content: line, + FileName: fileName, + LineNum: i + 1, + ExtractedLine: ndjson.Extract(line), + }) + } + return tagged +} + func discoverRepeatedPatterns(ctx context.Context, content []string) ([]pattern.DrainCluster, error) { drainParser, err := pattern.NewDrainParser() if err != nil { diff --git a/pkg/workspace/pipeline_ndjson_test.go b/pkg/workspace/pipeline_ndjson_test.go new file mode 100644 index 0000000..e7ea94d --- /dev/null +++ b/pkg/workspace/pipeline_ndjson_test.go @@ -0,0 +1,97 @@ +package workspace + +import ( + "context" + "encoding/json" + "path/filepath" + "strings" + "testing" + + "github.com/strrl/lapp/pkg/semantic" +) + +func TestDiscoverMixedTextAndNDJSONWorkspace(t *testing.T) { + dir := t.TempDir() + mustMkdir(t, filepath.Join(dir, "logs")) + textLines := []string{ + "2026-06-06 10:00:00 ERROR db timeout user=42", + "2026-06-06 10:00:01 ERROR db timeout user=43", + } + mustWrite(t, filepath.Join(dir, "logs", "app.log"), strings.Join(textLines, "\n")+"\n") + ndjsonLines := []string{ + `{"ts":"2026-06-06T10:00:02Z","severity":"ERROR","payload":{"message":"payment declined order=1001"}}`, + `{"ts":"2026-06-06T10:00:03Z","severity":"ERROR","payload":{"message":"payment declined order=1002"}}`, + } + unmatchedLine := `{"status_code":901,"details":{"state":"only_once_marker"}}` + mustWrite(t, filepath.Join(dir, "logs", "service.ndjson"), strings.Join(append(append([]string{}, ndjsonLines...), unmatchedLine), "\n")+"\n") + + result, err := Discover(context.Background(), DiscoveryConfig{ + Dir: dir, + RunID: "01900000-0000-7000-8000-000000000002", + Labeler: func(_ context.Context, _ semantic.Config, inputs []semantic.PatternInput) ([]semantic.SemanticLabel, error) { + labels := make([]semantic.SemanticLabel, 0, len(inputs)) + for _, input := range inputs { + labels = append(labels, semantic.SemanticLabel{ + PatternUUIDString: input.PatternUUIDString, + SemanticID: "pattern-" + input.PatternUUIDString[:8], + Description: "Labeled by test", + }) + } + return labels, nil + }, + }) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + if result.FileCount != 2 || result.LineCount != 5 || result.PatternCount != 2 || result.UnmatchedCount != 1 { + t.Fatalf("unexpected result: %+v", result) + } + + record, err := ReadDiscoveryRunRecord(dir, result.RunID) + if err != nil { + t.Fatalf("ReadDiscoveryRunRecord: %v", err) + } + + textPattern := findPatternForFile(record.Patterns, "app.log") + if textPattern == nil { + t.Fatalf("expected a pattern originating from app.log, got %+v", record.Patterns) + } + jsonPattern := findPatternForFile(record.Patterns, "service.ndjson") + if jsonPattern == nil { + t.Fatalf("expected a pattern originating from service.ndjson, got %+v", record.Patterns) + } + + runDir := DiscoveryRunDir(dir, result.RunID) + textSamples := mustRead(t, filepath.Join(runDir, "patterns", textPattern.DirName, "samples.log")) + assertContains(t, textSamples, "ERROR db timeout user=42") + + jsonSamples := mustRead(t, filepath.Join(runDir, "patterns", jsonPattern.DirName, "samples.log")) + sampleLines := strings.Split(strings.TrimSuffix(jsonSamples, "\n"), "\n") + if len(sampleLines) != len(ndjsonLines) { + t.Fatalf("expected %d NDJSON samples, got %d:\n%s", len(ndjsonLines), len(sampleLines), jsonSamples) + } + for i, line := range sampleLines { + var entry map[string]any + if err := json.Unmarshal([]byte(line), &entry); err != nil { + t.Fatalf("sample line %d is not a JSON object: %v\nline: %s", i+1, err, line) + } + if line != ndjsonLines[i] { + t.Fatalf("sample line %d is not the raw NDJSON line\ngot: %s\nwant: %s", i+1, line, ndjsonLines[i]) + } + } + + unmatchedSamples := mustRead(t, filepath.Join(runDir, "patterns", "unmatched", "samples.log")) + assertContains(t, unmatchedSamples, unmatchedLine) +} + +func findPatternForFile(patterns []PatternInfo, fileName string) *PatternInfo { + for i := range patterns { + for _, ref := range patterns[i].LineRefs { + if ref.FileName == fileName { + return &patterns[i] + } + } + } + return nil +} diff --git a/pkg/workspace/workspace.go b/pkg/workspace/workspace.go index 631b112..78f934e 100644 --- a/pkg/workspace/workspace.go +++ b/pkg/workspace/workspace.go @@ -12,6 +12,18 @@ type TaggedLine struct { Content string FileName string LineNum int + // ExtractedLine is the text fed to pattern mining for NDJSON entries. + // Empty for plain text entries, whose Content is mined directly. + ExtractedLine string `json:",omitempty"` +} + +// DrainLine returns the text used for pattern mining and template matching: +// the extracted line for NDJSON entries, otherwise the raw content. +func (t TaggedLine) DrainLine() string { + if t.ExtractedLine != "" { + return t.ExtractedLine + } + return t.Content } // LineRef identifies a line's location in a source file.