diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cfaf623a..515aeb43 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -196,6 +196,11 @@ jobs: - name: Lint with gavel uses: flanksource/gavel@4ec24346ade41681a5f1db1de47acd32c32e0879 # v0.0.83 + # gavel installs golangci-lint by resolving its stable release through the + # GitHub API; unauthenticated, a shared runner hits the rate limit and the + # install fails, so the read-only workflow token authenticates it. + env: + GITHUB_TOKEN: ${{ github.token }} with: args: lint --timeout 10m artifact-name: gavel-lint diff --git a/cmd/query/connections/browser_traces.go b/cmd/query/connections/browser_traces.go index 28f58cf5..e43fb727 100644 --- a/cmd/query/connections/browser_traces.go +++ b/cmd/query/connections/browser_traces.go @@ -2,16 +2,10 @@ package connections import ( "context" - "encoding/json" - "errors" - "fmt" "net/http" - "net/url" - "strconv" "strings" - "time" - "github.com/flanksource/clicky/rpc" + dbcontext "github.com/flanksource/commons-db/context" "github.com/flanksource/commons-db/models" "github.com/flanksource/commons-db/query/tracequery" ) @@ -21,70 +15,10 @@ func (h *connectionBrowserHandler) serveTraces(w http.ResponseWriter, r *http.Re http.Error(w, "connection does not support OpenSearch traces", 400) return } - if r.Header.Get("Accept") == "application/schema+json" { - writeJSON(w, rpc.SchemaForStruct(tracequery.Request{})) - return - } - request, err := browserTraceRequest(r) - if err != nil { - http.Error(w, err.Error(), 400) - return - } - request.Source = conn.ID.String() - ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) - defer cancel() - service := tracequery.Service{Read: func(ctx context.Context, request tracequery.Request, ids []string) ([]tracequery.Span, error) { - queryContext := withBrowserConnection(h.ctx.Wrap(ctx), conn) - return tracequery.ReadOpenSearch(queryContext, request, conn, ids) + service := tracequery.Service{Resolve: func(ctx context.Context, _ string) (dbcontext.Context, *models.Connection, error) { + return withBrowserConnection(h.ctx.Wrap(ctx), conn), conn, nil }} - var result any - if tail == "/traces" { - result, err = service.Search(ctx, request) - } else { - result, err = service.Trace(ctx, request, strings.TrimPrefix(tail, "/traces/")) - } - if err != nil { - status := http.StatusBadRequest - if errors.Is(err, tracequery.ErrTraceNotFound) { - status = http.StatusNotFound - } - http.Error(w, err.Error(), status) - return - } - writeJSON(w, result) -} - -func browserTraceRequest(r *http.Request) (tracequery.Request, error) { - schema := rpc.SchemaForStruct(tracequery.Request{}) - body := map[string]any{} - values, err := url.ParseQuery(r.URL.RawQuery) - if err != nil { - return tracequery.Request{}, err - } - for key, items := range values { - property, ok := schema.Properties[key] - if !ok || len(items) != 1 { - return tracequery.Request{}, fmt.Errorf("unknown or repeated trace parameter %q", key) - } - switch property.Type { - case "integer": - body[key], err = strconv.Atoi(items[0]) - case "object": - var modes map[string]string - err = json.Unmarshal([]byte(items[0]), &modes) - body[key] = modes - default: - body[key] = items[0] - } - if err != nil { - return tracequery.Request{}, fmt.Errorf("%s: %w", key, err) - } - } - data, err := json.Marshal(body) - if err != nil { - return tracequery.Request{}, err - } - var request tracequery.Request - err = json.Unmarshal(data, &request) - return request, err + traces := r.Clone(r.Context()) + traces.URL.Path, traces.URL.RawPath = strings.TrimPrefix(tail, "/traces"), "" + tracequery.Handler(service, conn.ID.String()).ServeHTTP(w, traces) } diff --git a/cmd/query/connections/browser_traces_integration_test.go b/cmd/query/connections/browser_traces_integration_test.go index 2795d8a9..6720f759 100644 --- a/cmd/query/connections/browser_traces_integration_test.go +++ b/cmd/query/connections/browser_traces_integration_test.go @@ -28,7 +28,16 @@ var _ = Describe("OpenSearch trace browser integration", func() { return } if strings.HasSuffix(r.URL.Path, "/_field_caps") { - Expect(json.NewEncoder(w).Encode(map[string]any{"fields": map[string]any{"@timestamp": map[string]any{"date": map[string]any{"searchable": true, "aggregatable": true}}}})).To(Succeed()) + // The span layout is discovered from every mapped field; the time + // range only inspects the date field. + mapping := map[string]string{"@timestamp": "date", "trace_id": "keyword", "span_id": "keyword", "parent_id": "keyword", "service_name": "keyword", "operation_name": "keyword", "duration_ms": "double"} + fields := map[string]any{} + for name, typ := range mapping { + if requested := r.URL.Query().Get("fields"); requested == "*" || requested == name { + fields[name] = map[string]any{typ: map[string]any{"searchable": true, "aggregatable": true}} + } + } + Expect(json.NewEncoder(w).Encode(map[string]any{"fields": fields})).To(Succeed()) return } if strings.Contains(r.URL.Path, "/_mapping/field/") { diff --git a/cmd/query/connections/browser_traces_test.go b/cmd/query/connections/browser_traces_test.go index 80075623..99e274a8 100644 --- a/cmd/query/connections/browser_traces_test.go +++ b/cmd/query/connections/browser_traces_test.go @@ -1,25 +1,12 @@ package connections import ( - "net/http" - "net/http/httptest" - "github.com/flanksource/clicky/rpc" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) var _ = Describe("OpenSearch trace HTTP contract", func() { - It("decodes typed trace filters and rejects malformed parameters", func() { - request, err := browserTraceRequest(httptest.NewRequest(http.MethodGet, "/traces?minSpans=2&services=%7B%22checkout%22:%22include%22%7D", nil)) - Expect(err).NotTo(HaveOccurred()) - Expect(request.MinSpans).To(Equal(2)) - Expect(string(request.Services["checkout"])).To(Equal("include")) - for _, query := range []string{"minSpans=oops", "unknown=value", "limit=1&limit=2", "services=invalid"} { - _, err := browserTraceRequest(httptest.NewRequest(http.MethodGet, "/traces?"+query, nil)) - Expect(err).To(HaveOccurred()) - } - }) It("documents search and detail with their typed response schemas", func() { spec := &rpc.OpenAPISpec{} AddConnectionsOpenAPI(spec) diff --git a/cmd/query/connections/service.go b/cmd/query/connections/service.go index 4674b312..5ef07586 100644 --- a/cmd/query/connections/service.go +++ b/cmd/query/connections/service.go @@ -312,7 +312,7 @@ func (s *Service) withVirtual(stored []*models.Connection, options ListOptions) stored = slices.DeleteFunc(stored, func(existing *models.Connection) bool { return existing.Namespace == connection.Namespace && existing.Name == connection.Name }) - if types := connectionTypeFilter(options); len(types) > 0 && !slices.Contains(types, connection.Type) && !(connection.Type == "" && len(connection.Warnings) > 0) { + if types := connectionTypeFilter(options); len(types) > 0 && !slices.Contains(types, connection.Type) && (connection.Type != "" || len(connection.Warnings) == 0) { continue } connection = cloneConnection(connection) diff --git a/cmd/query/connections/store_layers.go b/cmd/query/connections/store_layers.go index c16891be..91ce1558 100644 --- a/cmd/query/connections/store_layers.go +++ b/cmd/query/connections/store_layers.go @@ -97,5 +97,5 @@ func (p *StoreProvider) saveToLayer(ctx context.Context, c *models.Connection) ( if saved == nil { return nil, fmt.Errorf("saved connection %q cannot be resolved", c.Name) } - return p.project(*saved, true), nil + return p.project(*saved), nil } diff --git a/cmd/query/connections/store_provider.go b/cmd/query/connections/store_provider.go index 9fbeb7bd..7540b52a 100644 --- a/cmd/query/connections/store_provider.go +++ b/cmd/query/connections/store_provider.go @@ -77,10 +77,9 @@ func (p *StoreProvider) ListConnections() ([]*models.Connection, error) { if err != nil { return nil, err } - writable := p.Writable() listed := make([]*models.Connection, 0, len(entries)) for _, entry := range entries { - listed = append(listed, p.project(entry, writable)) + listed = append(listed, p.project(entry)) } return listed, nil } @@ -90,18 +89,17 @@ func (p *StoreProvider) ResolveConnection(_ dbcontext.Context, reference string) if err != nil || entry == nil { return nil, err } - return p.project(*entry, p.Writable()), nil + return p.project(*entry), nil } // project is entry's connection as the service sees it: a write-layer // connection is editable while the layer is writable; another layer's stays // virtual and read-only, so the service refuses to edit it. -func (p *StoreProvider) project(entry store.Entry, writable bool) *models.Connection { +func (p *StoreProvider) project(entry store.Entry) *models.Connection { connection := entry.Connection connection.Properties = maps.Clone(connection.Properties) if entry.Layer.Name == p.layer.Name || slices.Contains(p.writableLayers, entry.Layer.Name) { - writable = dirWritable(entry.Layer.Dir) - connection.Virtual, connection.ReadOnly = false, !writable + connection.Virtual, connection.ReadOnly = false, !dirWritable(entry.Layer.Dir) } else { connection.Virtual, connection.ReadOnly = true, true } @@ -165,7 +163,7 @@ func (p *StoreProvider) SaveConnection(ctx context.Context, connection *models.C if saved == nil { return nil, fmt.Errorf("connection %q was saved but the store does not resolve it", resource.Name) } - return p.project(*saved, true), nil + return p.project(*saved), nil } // DeleteConnection removes the write layer's file for the connection id names. diff --git a/cmd/query/go.mod b/cmd/query/go.mod index 78d57232..88a690e1 100644 --- a/cmd/query/go.mod +++ b/cmd/query/go.mod @@ -27,7 +27,6 @@ require ( github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.12.1 github.com/valkey-io/valkey-go v1.0.79 - go.yaml.in/yaml/v3 v3.0.5 gocloud.dev v0.46.0 golang.org/x/sync v0.24.0 gorm.io/gorm v1.31.2 @@ -361,6 +360,7 @@ require ( go.opentelemetry.io/otel/trace v1.47.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect go.yaml.in/yaml/v4 v4.0.0-rc.4 // indirect golang.org/x/crypto v0.58.0 // indirect golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3 // indirect diff --git a/cmd/query/localfiles/handler.go b/cmd/query/localfiles/handler.go index 1c65c7dd..ea2164e7 100644 --- a/cmd/query/localfiles/handler.go +++ b/cmd/query/localfiles/handler.go @@ -39,7 +39,7 @@ func Handler(prefix string, next http.Handler) http.Handler { } if r.Method != http.MethodGet { w.Header().Set("Allow", "GET") - http.Error(w, "method not allowed", 405) + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } listing, err := list(r.URL.Query().Get("path")) diff --git a/cmd/query/profiles/family.go b/cmd/query/profiles/family.go index 73295894..09f41fc7 100644 --- a/cmd/query/profiles/family.go +++ b/cmd/query/profiles/family.go @@ -106,7 +106,11 @@ func (s *Service) listSurfaces(ctx context.Context) ([]entity.DynamicEntitySpec, // Every instance listed here is documented with a run, and the profile // extension has already described one that has none — so listing it would // both advertise a run that can only fail and duplicate its surface. - if !advertisesRun(resolved.Profile) { + runs, err := advertisesRun(resolved.Profile) + if err != nil { + return nil, fmt.Errorf("profile surface %q: %w", profile.Name, err) + } + if !runs { continue } surface, err := s.profileSurface(resolved.Profile) diff --git a/cmd/query/profiles/family_test.go b/cmd/query/profiles/family_test.go index 63e5d1cf..02042df6 100644 --- a/cmd/query/profiles/family_test.go +++ b/cmd/query/profiles/family_test.go @@ -271,17 +271,13 @@ func TestProfileFamilyResolvesAProfileCreatedAfterTheRoutesWereBuilt(t *testing. // The served document is the family's paths laid over the profile extension's, // and the family fills any profile path the extension left empty. A trace or a -// legacy placeholder is left without a run on purpose — running it can only +// catalog-only profile is left without a run on purpose — running it can only // fail — so the family must not put one back, nor list its surface twice. func TestProfileFamilyAdvertisesARunOnlyForAProfileThatRuns(t *testing.T) { - legacy, err := ConvertLegacyProfile([]byte("name: legacy-sql\nkind: sql_xevent\n")) - if err != nil { - t.Fatal(err) - } capture := lookupProfile("Capture") capture.Trace = &query.TraceSpec{} query.RegisterProvider(familyLookupMock{}) - service, _ := newProfileServiceTest(t, lookupProfile("Spans"), capture, legacy) + service, _ := newProfileServiceTest(t, lookupProfile("Spans"), capture, catalogOnlyProfile("catalog-sql")) service.RegisterFamily() t.Cleanup(func() { entity.UnregisterDynamicEntityFamily(profileFamilyName) }) @@ -313,7 +309,7 @@ func TestProfileFamilyAdvertisesARunOnlyForAProfileThatRuns(t *testing.T) { Surfaces int } got := map[string]advertised{} - for _, key := range []string{"profile-spans", "profile-capture", "profile-legacy-sql"} { + for _, key := range []string{"profile-spans", "profile-capture", "profile-catalog-sql"} { _, run := spec.Paths["/api/v1/profile/"+key]["get"] _, start := spec.Paths["/api/v1/profile/"+key+"/sessions"]["post"] surfaces := 0 @@ -325,9 +321,9 @@ func TestProfileFamilyAdvertisesARunOnlyForAProfileThatRuns(t *testing.T) { got[key] = advertised{Run: run, SessionStart: start, Surfaces: surfaces} } want := map[string]advertised{ - "profile-spans": {Run: true, SessionStart: false, Surfaces: 1}, - "profile-capture": {Run: false, SessionStart: true, Surfaces: 1}, - "profile-legacy-sql": {Run: false, SessionStart: false, Surfaces: 1}, + "profile-spans": {Run: true, SessionStart: false, Surfaces: 1}, + "profile-capture": {Run: false, SessionStart: true, Surfaces: 1}, + "profile-catalog-sql": {Run: false, SessionStart: false, Surfaces: 1}, } for key, expected := range want { if got[key] != expected { diff --git a/cmd/query/profiles/legacy_migration.go b/cmd/query/profiles/legacy_migration.go deleted file mode 100644 index 4ff82269..00000000 --- a/cmd/query/profiles/legacy_migration.go +++ /dev/null @@ -1,417 +0,0 @@ -package profiles - -import ( - "fmt" - "os" - "path/filepath" - "slices" - "strings" - - "github.com/flanksource/commons-db/query" - yamlv3 "go.yaml.in/yaml/v3" - "sigs.k8s.io/yaml" -) - -const legacyTraceProvider = "legacy-trace" - -type legacyTraceParam struct { - Field string `json:"field,omitempty" yaml:"field,omitempty"` - Operator string `json:"operator,omitempty" yaml:"operator,omitempty"` - - // Format is parsed so an old file still loads, but it is not migrated: no - // version of the trace query builder ever read it. - Format string `json:"format,omitempty" yaml:"format,omitempty"` - - Template string `json:"template,omitempty" yaml:"template,omitempty"` - Clause string `json:"clause,omitempty" yaml:"clause,omitempty"` - Description string `json:"description,omitempty" yaml:"description,omitempty"` - Internal bool `json:"internal,omitempty" yaml:"internal,omitempty"` - Required bool `json:"required,omitempty" yaml:"required,omitempty"` -} - -type legacyTraceColumn struct { - Name string `json:"name" yaml:"name"` - Field string `json:"field" yaml:"field"` - Style string `json:"style,omitempty" yaml:"style,omitempty"` - Detail bool `json:"detail,omitempty" yaml:"detail,omitempty"` -} - -type legacyTraceProfile struct { - Name string `json:"name" yaml:"name"` - Kind string `json:"kind,omitempty" yaml:"kind,omitempty"` - Format string `json:"format,omitempty" yaml:"format,omitempty"` - Index string `json:"index,omitempty" yaml:"index,omitempty"` - DateField string `json:"dateField,omitempty" yaml:"dateField,omitempty"` - TraceIDField string `json:"traceIdField,omitempty" yaml:"traceIdField,omitempty"` - SpanIDField string `json:"spanIdField,omitempty" yaml:"spanIdField,omitempty"` - ParentIDField string `json:"parentIdField,omitempty" yaml:"parentIdField,omitempty"` - ParentRefType string `json:"parentRefType,omitempty" yaml:"parentRefType,omitempty"` - ServiceField string `json:"serviceField,omitempty" yaml:"serviceField,omitempty"` - OperationField string `json:"operationField,omitempty" yaml:"operationField,omitempty"` - StatusFields []string `json:"statusFields,omitempty" yaml:"statusFields,omitempty"` - SelectFields []string `json:"selectFields,omitempty" yaml:"selectFields,omitempty"` - SourceExcludes []string `json:"sourceExcludes,omitempty" yaml:"sourceExcludes,omitempty"` - Imports []string `json:"imports,omitempty" yaml:"imports,omitempty"` - Defaults map[string]any `json:"defaults,omitempty" yaml:"defaults,omitempty"` - Params map[string]legacyTraceParam `json:"params,omitempty" yaml:"params,omitempty"` - Aliases orderedLegacyAliases `json:"-" yaml:"aliases,omitempty"` - Ignore []string `json:"ignore,omitempty" yaml:"ignore,omitempty"` - Columns []legacyTraceColumn `json:"columns,omitempty" yaml:"columns,omitempty"` - Filters []query.FilterDef `json:"filters,omitempty" yaml:"filters,omitempty"` - SQL map[string]any `json:"sql,omitempty" yaml:"sql,omitempty"` - Kubernetes map[string]any `json:"kubernetes,omitempty" yaml:"kubernetes,omitempty"` - Arthas map[string]any `json:"arthas,omitempty" yaml:"arthas,omitempty"` - Replay map[string]any `json:"replay,omitempty" yaml:"replay,omitempty"` -} - -type orderedLegacyAliases []query.AliasDef - -func (a *orderedLegacyAliases) UnmarshalYAML(node *yamlv3.Node) error { - if node.Kind != yamlv3.MappingNode { - return fmt.Errorf("aliases must be a mapping") - } - for index := 0; index < len(node.Content); index += 2 { - name, spec := node.Content[index].Value, node.Content[index+1] - // An alias is written either as a bare expression or as a mapping with a - // cel key. Both forms are in the wild, and the shorthand is the common - // one, so a profile using it must not be read as malformed. - if spec.Kind == yamlv3.ScalarNode { - *a = append(*a, query.AliasDef{Name: name, CEL: spec.Value}) - continue - } - var value struct { - CEL string `yaml:"cel"` - } - if err := spec.Decode(&value); err != nil { - return fmt.Errorf("alias %q: %w", name, err) - } - *a = append(*a, query.AliasDef{Name: name, CEL: value.CEL}) - } - return nil -} - -type profileMigration struct { - path string - data []byte -} - -func (s *FileStore) migrateLegacyTraceProfiles() error { - entries, err := os.ReadDir(s.Dir) - if os.IsNotExist(err) { - return nil - } - if err != nil { - return fmt.Errorf("read profiles dir %q for migration: %w", s.Dir, err) - } - migrations := make([]profileMigration, 0) - for _, entry := range entries { - if entry.IsDir() || !isYAML(entry.Name()) { - continue - } - path := filepath.Join(s.Dir, entry.Name()) - migration, ok, err := legacyProfileMigration(path) - if err != nil { - return err - } - if ok { - migrations = append(migrations, migration) - } - } - for _, migration := range migrations { - if err := replaceProfileFile(migration); err != nil { - return err - } - } - return nil -} - -func legacyProfileMigration(path string) (profileMigration, bool, error) { - data, err := os.ReadFile(path) - if err != nil { - return profileMigration{}, false, fmt.Errorf("read profile %q for migration: %w", path, err) - } - var identity struct { - Profile string `json:"profile"` - Name string `json:"name"` - Provider query.ProviderConfig `json:"provider"` - } - if err := yaml.Unmarshal(data, &identity); err != nil { - return profileMigration{}, false, fmt.Errorf("parse profile %q for migration: %w", path, err) - } - if identity.Profile != "" { - if identity.Provider.Type != legacyTraceProvider { - return profileMigration{}, false, nil - } - var current query.Profile - if err := yaml.Unmarshal(data, ¤t); err != nil { - return profileMigration{}, false, fmt.Errorf("parse interim trace profile %q: %w", path, err) - } - source, _ := current.Provider.Options["source"].(string) - kind, _ := current.Provider.Options["kind"].(string) - if source == "" || kind != "opensearch" && kind != "import" { - return profileMigration{}, false, nil - } - converted, err := convertLegacyTraceSource(source) - if err != nil { - return profileMigration{}, false, fmt.Errorf("convert interim trace profile %q: %w", path, err) - } - converted.Provider.Connection = current.Provider.Connection - data, err := yaml.Marshal(converted) - if err != nil { - return profileMigration{}, false, fmt.Errorf("marshal migrated trace profile %q: %w", path, err) - } - return profileMigration{path: path, data: data}, true, nil - } - if identity.Name == "" { - return profileMigration{}, false, nil - } - profile, err := convertLegacyTraceSource(string(data)) - if err != nil { - return profileMigration{}, false, fmt.Errorf("convert legacy trace profile %q: %w", path, err) - } - converted, err := yaml.Marshal(profile) - if err != nil { - return profileMigration{}, false, fmt.Errorf("marshal migrated trace profile %q: %w", path, err) - } - return profileMigration{path: path, data: converted}, true, nil -} - -// ConvertLegacyProfile turns one legacy trace-profile document into a Profile. -// -// FileStore migrates a directory in place, which is the right shape for user -// files on disk. An application whose built-in profiles are embedded has no -// directory to migrate and needs the same mapping applied at load time, so the -// conversion is available on its own rather than only as a side effect of -// opening a store. -func ConvertLegacyProfile(source []byte) (query.Profile, error) { - return convertLegacyTraceSource(string(source)) -} - -func convertLegacyTraceSource(source string) (query.Profile, error) { - var legacy legacyTraceProfile - if err := yamlv3.Unmarshal([]byte(source), &legacy); err != nil { - return query.Profile{}, fmt.Errorf("parse legacy trace source: %w", err) - } - return convertLegacyTraceProfile(legacy, source) -} - -func convertLegacyTraceProfile(legacy legacyTraceProfile, source string) (query.Profile, error) { - if strings.TrimSpace(legacy.Name) == "" { - return query.Profile{}, fmt.Errorf("name is required") - } - params := make([]query.ParamDef, 0, len(legacy.Params)) - for _, name := range sortedKeys(legacy.Params) { - param := legacy.Params[name] - def := query.ParamDef{ - Name: name, Description: param.Description, Required: param.Required, Template: param.Template, - } - if value, ok := legacy.Defaults[name]; ok { - def.Default = value - def.Type = legacyParamType(value) - } - params = append(params, def) - } - columns := make([]query.ColumnDef, len(legacy.Columns)) - for i, column := range legacy.Columns { - columns[i] = query.ColumnDef{ - Name: column.Name, CEL: column.Field, Hidden: column.Detail, Style: column.Style, - } - } - provider := query.ProviderConfig{Type: legacyTraceProvider, Options: map[string]any{ - "kind": legacyTraceKind(legacy), "source": source, - }} - profileQuery := "" - kind := legacyTraceKind(legacy) - if kind == "kubernetes" { - config, err := legacyKubernetesConfigFor(legacy) - if err != nil { - return query.Profile{}, err - } - provider = query.ProviderConfig{Type: "k8s", Options: config.Options} - profileQuery = config.Query - if config.NeedsNamespaceParam { - params = ensureParam(params, query.ParamDef{ - Name: "namespace", Description: "Kubernetes namespace to read logs from", - }) - } - } - if kind == "opensearch" || kind == "import" { - search, err := legacyOpenTelemetrySearch(legacy.Params) - if err != nil { - return query.Profile{}, err - } - options := map[string]any{} - // An import inherits every index and field setting from its parent, so - // only its own filters travel with it. - if kind == "opensearch" { - options = legacyOpenTelemetryOptions(legacy) - } - if search != nil { - options["search"] = search - } - provider = query.ProviderConfig{Type: "opentelemetry", Options: options} - } - return query.Profile{ - Name: legacy.Name, Imports: legacy.Imports, Provider: provider, - Query: profileQuery, - Params: params, Columns: columns, Aliases: []query.AliasDef(legacy.Aliases), Ignore: legacy.Ignore, - Filters: legacy.Filters, - }, nil -} - -// ensureParam appends def unless the profile already declares that name, so a -// parameter the conversion needs never shadows one the author wrote. -func ensureParam(params []query.ParamDef, def query.ParamDef) []query.ParamDef { - for _, existing := range params { - if existing.Name == def.Name { - return params - } - } - return append(params, def) -} - -// legacyKubernetesWorkloadKinds maps the short workload references a legacy -// `kubernetes.target` uses onto the kinds the k8s log provider accepts. -var legacyKubernetesWorkloadKinds = map[string]string{ - "po": "Pod", "pod": "Pod", "pods": "Pod", - "deploy": "Deployment", "deployment": "Deployment", "deployments": "Deployment", - "sts": "StatefulSet", "statefulset": "StatefulSet", "statefulsets": "StatefulSet", - "ds": "DaemonSet", "daemonset": "DaemonSet", "daemonsets": "DaemonSet", -} - -type legacyKubernetesConfig struct { - Query string - Options map[string]any - NeedsNamespaceParam bool -} - -// legacyKubernetesConfigFor moves target scope into the query grammar while -// preserving log-reading settings as provider options. -func legacyKubernetesConfigFor(legacy legacyTraceProfile) (legacyKubernetesConfig, error) { - config := legacyKubernetesConfig{Options: map[string]any{}} - selector := make([]string, 0, 3) - var name string - options := map[string]any{} - if target, _ := legacy.Kubernetes["target"].(string); target != "" { - shorthand, targetName, found := strings.Cut(target, "/") - if !found || targetName == "" { - return config, fmt.Errorf("kubernetes.target %q is not a kind/name reference", target) - } - kind, ok := legacyKubernetesWorkloadKinds[strings.ToLower(shorthand)] - if !ok { - return config, fmt.Errorf("kubernetes.target %q has an unsupported kind %q", target, shorthand) - } - selector = append(selector, "kind="+kind) - name = targetName - } - // The legacy reader resolved an empty namespace from the active kubecontext. - // The provider requires one, so it becomes a parameter the caller supplies - // rather than an ambient value the profile silently inherits. - if namespace, _ := legacy.Kubernetes["namespace"].(string); namespace != "" { - selector = append(selector, "namespace="+namespace) - } else { - selector = append(selector, "namespace={{.params.namespace}}") - config.NeedsNamespaceParam = true - } - if name != "" { - selector = append(selector, "name="+name) - } - if since, _ := legacy.Kubernetes["since"].(string); since != "" { - options["start"] = "now-" + since - } - if tail, ok := legacy.Kubernetes["tail"]; ok { - options["limit"] = fmt.Sprintf("%v", tail) - } - // Fail loudly on the settings that have no home, rather than dropping them - // and quietly widening what the profile reads. - for _, unsupported := range []string{"selector", "container", "grep", "previous"} { - if value, ok := legacy.Kubernetes[unsupported]; ok && value != nil && value != "" { - return config, fmt.Errorf("kubernetes.%s is not supported by the k8s provider", unsupported) - } - } - config.Query = strings.Join(selector, " ") - config.Options = options - return config, nil -} - -func legacyOpenTelemetryOptions(legacy legacyTraceProfile) map[string]any { - options := map[string]any{ - "format": legacy.Format, "index": legacy.Index, "dateField": legacy.DateField, - "traceIdField": legacy.TraceIDField, "spanIdField": legacy.SpanIDField, - "parentIdField": legacy.ParentIDField, "parentRefType": legacy.ParentRefType, - "serviceField": legacy.ServiceField, "operationField": legacy.OperationField, - "statusFields": legacy.StatusFields, "selectFields": legacy.SelectFields, - "sourceExcludes": legacy.SourceExcludes, - } - for key, value := range options { - switch typed := value.(type) { - case string: - if typed == "" { - delete(options, key) - } - case []string: - if len(typed) == 0 { - delete(options, key) - } - case map[string]any: - if len(typed) == 0 { - delete(options, key) - } - } - } - return options -} - -func legacyTraceKind(profile legacyTraceProfile) string { - if profile.Kind != "" { - return profile.Kind - } - switch { - case profile.SQL != nil: - return "sql" - case profile.Kubernetes != nil: - return "kubernetes" - case profile.Arthas != nil: - return "arthas" - case profile.Index != "" || profile.Format != "": - return "opensearch" - case profile.Replay != nil: - return "replay" - case len(profile.Imports) > 0: - return "import" - default: - return "unknown" - } -} - -func legacyParamType(value any) query.ParamType { - switch value.(type) { - case bool: - return query.ParamTypeBoolean - case float32, float64, int, int32, int64: - return query.ParamTypeNumber - default: - return query.ParamTypeString - } -} - -func replaceProfileFile(migration profileMigration) error { - temporary := migration.path + ".migrating" - if err := os.WriteFile(temporary, migration.data, 0o600); err != nil { - return fmt.Errorf("write migrated profile %q: %w", migration.path, err) - } - if err := os.Rename(temporary, migration.path); err != nil { - _ = os.Remove(temporary) - return fmt.Errorf("replace migrated profile %q: %w", migration.path, err) - } - return nil -} - -func sortedKeys[V any](values map[string]V) []string { - keys := make([]string, 0, len(values)) - for key := range values { - keys = append(keys, key) - } - slices.Sort(keys) - return keys -} diff --git a/cmd/query/profiles/legacy_migration_test.go b/cmd/query/profiles/legacy_migration_test.go deleted file mode 100644 index 227ca9ae..00000000 --- a/cmd/query/profiles/legacy_migration_test.go +++ /dev/null @@ -1,304 +0,0 @@ -package profiles - -import ( - "fmt" - "reflect" - "testing" - - dbcontext "github.com/flanksource/commons-db/context" - "github.com/flanksource/commons-db/query" - "github.com/flanksource/commons-db/query/esdsl" -) - -func TestLegacyTraceKind(t *testing.T) { - tests := []struct { - name string - profile legacyTraceProfile - want string - }{ - {name: "explicit", profile: legacyTraceProfile{Kind: "watch"}, want: "watch"}, - {name: "sql", profile: legacyTraceProfile{SQL: map[string]any{}}, want: "sql"}, - {name: "kubernetes", profile: legacyTraceProfile{Kubernetes: map[string]any{}}, want: "kubernetes"}, - {name: "arthas", profile: legacyTraceProfile{Arthas: map[string]any{}}, want: "arthas"}, - {name: "opensearch", profile: legacyTraceProfile{Index: "traces-*"}, want: "opensearch"}, - {name: "replay", profile: legacyTraceProfile{Replay: map[string]any{}}, want: "replay"}, - {name: "import", profile: legacyTraceProfile{Imports: []string{"base"}}, want: "import"}, - {name: "unknown", profile: legacyTraceProfile{}, want: "unknown"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := legacyTraceKind(tt.profile); got != tt.want { - t.Fatalf("legacyTraceKind() = %q, want %q", got, tt.want) - } - }) - } -} - -// Legacy trace profiles carry noise filters that are the whole reason some of -// them are readable, so conversion has to bring them across rather than quietly -// widening the result. -func TestLegacyFiltersSurviveConversion(t *testing.T) { - source := ` -name: example app logs -kubernetes: - since: 24h -aliases: - logger: 'map_string(_fields, "log.logger", "logger")' -filters: - - name: drop-access-log-2xx - description: Drop successful HTTP access log lines - exclude: true - hidden: true - fields: - logger: 'logger == "AccessLog"' - status_2xx: 'string(message).contains("=> 2")' - - name: errors-and-warnings - fields: - level: 'level == "ERROR"' -` - profile, err := convertLegacyTraceSource(source) - if err != nil { - t.Fatalf("convertLegacyTraceSource() error = %v", err) - } - if len(profile.Filters) != 2 { - t.Fatalf("got %d filters, want 2", len(profile.Filters)) - } - - dropped := profile.Filters[0] - if dropped.Name != "drop-access-log-2xx" { - t.Errorf("filter name = %q, want %q", dropped.Name, "drop-access-log-2xx") - } - if dropped.Description != "Drop successful HTTP access log lines" { - t.Errorf("filter description = %q", dropped.Description) - } - if !dropped.Exclude || !dropped.Hidden { - t.Errorf("exclude/hidden = %v/%v, want true/true", dropped.Exclude, dropped.Hidden) - } - want := map[string]string{ - "logger": `logger == "AccessLog"`, - "status_2xx": `string(message).contains("=> 2")`, - } - if !reflect.DeepEqual(dropped.Fields, want) { - t.Errorf("fields = %#v, want %#v", dropped.Fields, want) - } - - if quick := profile.Filters[1]; quick.Hidden || quick.Exclude { - t.Errorf("quick filter should stay togglable and inclusive, got hidden=%v exclude=%v", - quick.Hidden, quick.Exclude) - } -} - -func TestLegacyKubernetesTargetBecomesK8sProvider(t *testing.T) { - tests := []struct { - name string - source string - wantKind string - wantName string - }{ - {name: "deployment", source: "target: deployment/example-app", wantKind: "Deployment", wantName: "example-app"}, - {name: "pod shorthand", source: "target: po/cycle-0", wantKind: "Pod", wantName: "cycle-0"}, - {name: "statefulset", source: "target: sts/cycle", wantKind: "StatefulSet", wantName: "cycle"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - profile, err := convertLegacyTraceSource("name: logs\nkubernetes:\n " + tt.source + "\n") - if err != nil { - t.Fatalf("convertLegacyTraceSource() error = %v", err) - } - if profile.Provider.Type != "k8s" { - t.Fatalf("provider = %q, want k8s", profile.Provider.Type) - } - wantQuery := fmt.Sprintf("kind=%s namespace={{.params.namespace}} name=%s", tt.wantKind, tt.wantName) - if profile.Query != wantQuery { - t.Errorf("query = %q, want %q", profile.Query, wantQuery) - } - if len(profile.Provider.Options) != 0 { - t.Errorf("provider options = %#v, want no target options", profile.Provider.Options) - } - // The legacy reader took the namespace from the active kubecontext; - // the query grammar needs it stated, so it becomes a parameter. - if len(profile.Params) != 1 || profile.Params[0].Name != "namespace" { - t.Errorf("params = %#v, want the namespace parameter", profile.Params) - } - }) - } -} - -func TestLegacyKubernetesSinceAndTail(t *testing.T) { - profile, err := convertLegacyTraceSource("name: base\nkubernetes:\n since: 24h\n tail: 5000\n") - if err != nil { - t.Fatalf("convertLegacyTraceSource() error = %v", err) - } - if got := profile.Provider.Options["start"]; got != "now-24h" { - t.Errorf("start = %v, want now-24h", got) - } - if got := profile.Provider.Options["limit"]; got != "5000" { - t.Errorf("limit = %v, want \"5000\"", got) - } - if profile.Query != "namespace={{.params.namespace}}" { - t.Errorf("query = %q, want a broad namespace selector", profile.Query) - } -} - -func TestLegacyKubernetesExplicitNamespaceUsesQueryGrammar(t *testing.T) { - profile, err := convertLegacyTraceSource("name: logs\nkubernetes:\n target: ds/agent\n namespace: monitoring\n") - if err != nil { - t.Fatalf("convertLegacyTraceSource() error = %v", err) - } - if profile.Query != "kind=DaemonSet namespace=monitoring name=agent" { - t.Errorf("query = %q, want the explicit namespace selector", profile.Query) - } - if len(profile.Params) != 0 { - t.Errorf("params = %#v, want no namespace parameter", profile.Params) - } -} - -func TestLegacyKubernetesUnsupportedSettingsFailLoudly(t *testing.T) { - for _, unsupported := range []string{"selector: app=example-app", "container: example-app", "grep: ERROR", "previous: true"} { - if _, err := convertLegacyTraceSource("name: logs\nkubernetes:\n target: po/x\n " + unsupported + "\n"); err == nil { - t.Errorf("converting kubernetes.%s should fail rather than silently widen the read", unsupported) - } - } -} - -// A legacy column carries a per-cell style expression and a detail flag; both -// have direct homes on the converted column. -func TestLegacyColumnStyleAndDetail(t *testing.T) { - profile, err := convertLegacyTraceSource(` -name: styled -index: traces-* -columns: - - name: Level - field: level - style: 'level == "ERROR" ? "text-red-500" : ""' - - name: StackTrace - field: exception - detail: true -`) - if err != nil { - t.Fatalf("convertLegacyTraceSource() error = %v", err) - } - want := []query.ColumnDef{ - {Name: "Level", CEL: "level", Style: `level == "ERROR" ? "text-red-500" : ""`}, - {Name: "StackTrace", CEL: "exception", Hidden: true}, - } - if !reflect.DeepEqual(profile.Columns, want) { - t.Fatalf("columns = %#v, want %#v", profile.Columns, want) - } -} - -// Aliases appear in the wild both as a bare expression and as a mapping with a -// cel key; the shorthand is the common one. -func TestLegacyScalarAliasShorthand(t *testing.T) { - profile, err := convertLegacyTraceSource(` -name: aliases -index: traces-* -aliases: - level: 'map_string(_fields, "log.level", "level")' - logger: - cel: 'map_string(_fields, "log.logger")' -`) - if err != nil { - t.Fatalf("convertLegacyTraceSource() error = %v", err) - } - want := []query.AliasDef{ - {Name: "level", CEL: `map_string(_fields, "log.level", "level")`}, - {Name: "logger", CEL: `map_string(_fields, "log.logger")`}, - } - if !reflect.DeepEqual(profile.Aliases, want) { - t.Fatalf("aliases = %#v, want %#v", profile.Aliases, want) - } -} - -func TestLegacyOpenTelemetrySearch(t *testing.T) { - search, err := legacyOpenTelemetrySearch(map[string]legacyTraceParam{ - "namespace": {Field: "process.serviceName"}, - "levels": {Field: "level", Operator: "terms"}, - "phrase": {Field: "message", Operator: "match_phrase", Clause: "must"}, - "exclude": {Field: "env", Operator: "wildcard", Clause: "must_not"}, - "lucene": {Field: "body", Operator: "query_string"}, - "errored": {Field: "error", Operator: "exists"}, - "hidden": {Field: "tenant", Internal: true}, - }) - if err != nil { - t.Fatalf("legacyOpenTelemetrySearch() error: %v", err) - } - want := &esdsl.Search{Query: &esdsl.Condition{Op: esdsl.OpBool, Conditions: []esdsl.Condition{ - {Op: esdsl.OpExists, Field: "error", When: "errored"}, - {Op: esdsl.OpWildcard, Field: "env", Occur: esdsl.OccurMustNot, Value: esdsl.Param("exclude"), Optional: true}, - {Op: esdsl.OpTerms, Field: "level", Value: esdsl.Param("levels"), Optional: true}, - {Op: esdsl.OpQueryString, Fields: []string{"body"}, Value: esdsl.Param("lucene"), Optional: true}, - {Op: esdsl.OpTerms, Field: "process.serviceName", Value: esdsl.Param("namespace"), Optional: true}, - {Op: esdsl.OpMatchPhrase, Field: "message", Occur: esdsl.OccurMust, Value: esdsl.Param("phrase"), Optional: true}, - }}} - if !reflect.DeepEqual(search, want) { - t.Fatalf("legacyOpenTelemetrySearch() =\n%+v\nwant\n%+v", search.Query.Conditions, want.Query.Conditions) - } -} - -func TestLegacyOpenTelemetrySearchRejections(t *testing.T) { - tests := []struct { - name string - params map[string]legacyTraceParam - want string - }{ - { - name: "unknown operator", - params: map[string]legacyTraceParam{"level": {Field: "level", Operator: "span_near"}}, - want: `param "level": unsupported operator "span_near"`, - }, - { - name: "missing field", - params: map[string]legacyTraceParam{"level": {Operator: "term"}}, - want: `param "level": field is required`, - }, - { - name: "unknown clause", - params: map[string]legacyTraceParam{"level": {Field: "level", Clause: "maybe"}}, - want: `query.conditions[0]: unknown occur "maybe"`, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := legacyOpenTelemetrySearch(tt.params) - if err == nil || err.Error() != tt.want { - t.Fatalf("legacyOpenTelemetrySearch() error = %v, want %q", err, tt.want) - } - }) - } -} - -// An internal parameter contributes no condition but keeps its declaration, so -// supplying one still fails instead of silently widening the query. -func TestLegacyInternalParamKeepsItsDeclaration(t *testing.T) { - profile, err := convertLegacyTraceProfile(legacyTraceProfile{ - Name: "traces", Index: "traces-*", - Params: map[string]legacyTraceParam{ - "tenant": {Field: "tenant", Internal: true}, - "namespace": {Field: "process.serviceName"}, - }, - }, "") - if err != nil { - t.Fatalf("convertLegacyTraceProfile() error: %v", err) - } - names := make([]string, 0, len(profile.Params)) - for _, param := range profile.Params { - names = append(names, param.Name) - } - if !reflect.DeepEqual(names, []string{"namespace", "tenant"}) { - t.Fatalf("migrated params = %v, want [namespace tenant]", names) - } - search := profile.Provider.Options["search"].(*esdsl.Search) - if len(search.Query.Conditions) != 1 || search.Query.Conditions[0].Field != "process.serviceName" { - t.Fatalf("migrated search conditions = %+v, want only process.serviceName", search.Query.Conditions) - } -} - -func TestLegacyTraceProviderFailsLoudly(t *testing.T) { - _, err := legacyTraceProfileProvider{}.Execute(dbcontext.New(), query.ProviderRequest{ - Options: map[string]any{"kind": "sql"}, - }) - if err == nil || err.Error() != `legacy trace profile kind "sql" is catalog-compatible but is not executable by the query engine` { - t.Fatalf("unexpected legacy provider error: %v", err) - } -} diff --git a/cmd/query/profiles/legacy_provider.go b/cmd/query/profiles/legacy_provider.go deleted file mode 100644 index 39ba4eb6..00000000 --- a/cmd/query/profiles/legacy_provider.go +++ /dev/null @@ -1,23 +0,0 @@ -package profiles - -import ( - "fmt" - - dbcontext "github.com/flanksource/commons-db/context" - "github.com/flanksource/commons-db/query" -) - -type legacyTraceProfileProvider struct{} - -func (legacyTraceProfileProvider) Type() string { return legacyTraceProvider } - -func (legacyTraceProfileProvider) Execute(_ dbcontext.Context, request query.ProviderRequest) ([]query.Row, error) { - return nil, fmt.Errorf( - "legacy trace profile kind %q is catalog-compatible but is not executable by the query engine", - request.Options["kind"], - ) -} - -func init() { - query.RegisterProvider(legacyTraceProfileProvider{}) -} diff --git a/cmd/query/profiles/legacy_search.go b/cmd/query/profiles/legacy_search.go deleted file mode 100644 index 14970ca4..00000000 --- a/cmd/query/profiles/legacy_search.go +++ /dev/null @@ -1,79 +0,0 @@ -package profiles - -import ( - "fmt" - - "github.com/flanksource/commons-db/query/esdsl" -) - -// legacyOperators maps the legacy per-parameter operator vocabulary onto the -// search specification's. The legacy builder split every value on commas and -// emitted one clause per part; the list operators keep that, because the -// specification's terms binding splits the same way. A phrase, pattern or query -// string now receives its value whole, which is what a comma inside one always -// meant. -var legacyOperators = map[string]esdsl.Operator{ - "": esdsl.OpTerms, - "term": esdsl.OpTerms, - "terms": esdsl.OpTerms, - "match_phrase": esdsl.OpMatchPhrase, - "wildcard": esdsl.OpWildcard, - "query_string": esdsl.OpQueryString, - "exists": esdsl.OpExists, -} - -// legacyOpenTelemetrySearch converts the legacy filter parameters into the -// structured search specification the opentelemetry provider compiles. Every -// condition is optional, matching the legacy builder's rule that an unsupplied -// parameter contributes no clause. -// -// Internal parameters are left out: supplying one was an error, and one left -// unsupplied contributed nothing either way. Their ParamDefs survive, so -// supplying one still fails — now because nothing in the specification -// references it. -func legacyOpenTelemetrySearch(params map[string]legacyTraceParam) (*esdsl.Search, error) { - conditions := make([]esdsl.Condition, 0, len(params)) - for _, name := range sortedKeys(params) { - param := params[name] - if param.Internal { - continue - } - condition, err := legacyCondition(name, param) - if err != nil { - return nil, fmt.Errorf("param %q: %w", name, err) - } - conditions = append(conditions, condition) - } - if len(conditions) == 0 { - return nil, nil - } - search := &esdsl.Search{Query: &esdsl.Condition{Op: esdsl.OpBool, Conditions: conditions}} - if err := search.Validate(); err != nil { - return nil, err - } - return search, nil -} - -func legacyCondition(name string, param legacyTraceParam) (esdsl.Condition, error) { - operator, supported := legacyOperators[param.Operator] - if !supported { - return esdsl.Condition{}, fmt.Errorf("unsupported operator %q", param.Operator) - } - if param.Field == "" { - return esdsl.Condition{}, fmt.Errorf("field is required") - } - condition := esdsl.Condition{Op: operator, Field: param.Field, Occur: esdsl.Occur(param.Clause)} - // exists takes no operand, so the parameter gates the clause instead of - // supplying it — the legacy builder emitted it only when one was supplied. - if operator == esdsl.OpExists { - condition.When = name - return condition, nil - } - if operator == esdsl.OpQueryString { - condition.Fields = []string{param.Field} - condition.Field = "" - } - condition.Value = esdsl.Param(name) - condition.Optional = true - return condition, nil -} diff --git a/cmd/query/profiles/openapi.go b/cmd/query/profiles/openapi.go index de2ad112..0ee8f517 100644 --- a/cmd/query/profiles/openapi.go +++ b/cmd/query/profiles/openapi.go @@ -170,12 +170,22 @@ func mergeStoredProfiles(ctx context.Context, spec *rpc.OpenAPISpec, store profi // advertisesRun reports whether a profile gets a run operation: a single-shot // read a surface may execute the moment the profile is opened. A trace or top // profile is refused by that read and started as a session instead, and a -// legacy trace placeholder is listed but refused by every execution, so for -// them a run would be an entry point that can only fail. -func advertisesRun(profile query.Profile) bool { - return profile.Kind() == query.KindQuery && profile.Provider.Type != legacyTraceProvider +// profile whose provider is not query.RunnableProvider-runnable is listed but +// refused by every execution, so for them a run would be an entry point that +// can only fail. +func advertisesRun(profile query.Profile) (bool, error) { + if profile.Kind() != query.KindQuery || profile.Provider.Type == retiredLegacyTraceProvider { + return false, nil + } + return query.ProviderRunnable(profile.Provider.Type) } +// retiredLegacyTraceProvider is the provider type the removed legacy trace +// migration stored converted profiles under. Its provider is gone, so such a +// profile never runs, but an upgraded store may still hold one: it stays +// listed without a run rather than failing every profile surface. +const retiredLegacyTraceProvider = "legacy-trace" + func addProfileToSpec(spec *rpc.OpenAPISpec, profile query.Profile) error { entityName := SurfaceKey(profile.Name) path := "/api/v1/profile/" + entityName @@ -316,7 +326,11 @@ func addProfileToSpec(spec *rpc.OpenAPISpec, profile query.Profile) error { // and points the caller at ExecuteStream. Advertising a run operation anyway // is how a generic surface ends up executing one the moment the profile is // opened — so the session start below is the only entry point it gets. - if advertisesRun(profile) { + runs, err := advertisesRun(profile) + if err != nil { + return fmt.Errorf("profile %q: %w", profile.Name, err) + } + if runs { spec.Paths[path] = rpc.OpenAPIPath{"get": { Summary: "Run " + profile.Name, Description: "Execute the stored query profile", diff --git a/cmd/query/profiles/openapi_test.go b/cmd/query/profiles/openapi_test.go index a2a03722..f472ad38 100644 --- a/cmd/query/profiles/openapi_test.go +++ b/cmd/query/profiles/openapi_test.go @@ -3,12 +3,14 @@ package profiles import ( "context" "fmt" + "slices" "sort" "strings" "testing" "github.com/flanksource/clicky/entity" "github.com/flanksource/clicky/rpc" + dbcontext "github.com/flanksource/commons-db/context" "github.com/flanksource/commons-db/query" ) @@ -97,7 +99,8 @@ func TestMergeStoredProfilesTracksStoreImmediately(t *testing.T) { func TestProfileOpenAPIPreservesMappedPagingAndTimeRoles(t *testing.T) { spec := &rpc.OpenAPISpec{Paths: map[string]rpc.OpenAPIPath{}, Clicky: &rpc.ClickySpecMeta{}} if err := addProfileToSpec(spec, query.Profile{ - Name: "mapped", + Name: "mapped", + Provider: query.ProviderConfig{Type: "postgres"}, Params: []query.ParamDef{ {Name: "page_size", Type: query.ParamTypeNumber, Role: query.ParamRoleLimit}, {Name: "skip", Type: query.ParamTypeNumber, Role: query.ParamRoleOffset}, @@ -567,25 +570,67 @@ func TestProfileOpenAPITopOffersOnlySessionStart(t *testing.T) { } } -// A legacy trace placeholder is catalogued so it can be listed, but its -// provider refuses every run. Advertising a run operation would hand a generic -// surface an entry point that can only fail, so it gets none — neither a run -// nor a session start, since it is not a trace the engine can capture either. -func TestProfileOpenAPILegacyTraceOffersNoOperation(t *testing.T) { - profile, err := ConvertLegacyProfile([]byte("name: sql\nkind: sql_xevent\ndescription: Capture SQL statements.\n")) - if err != nil { - t.Fatal(err) - } +// catalogOnlyProvider lists its profiles but refuses every execution. +type catalogOnlyProvider struct{} + +const catalogOnlyProviderType = "catalog-only" + +func (catalogOnlyProvider) Type() string { return catalogOnlyProviderType } + +func (catalogOnlyProvider) Runnable() bool { return false } + +func (catalogOnlyProvider) Execute(dbcontext.Context, query.ProviderRequest) ([]query.Row, error) { + return nil, fmt.Errorf("%s profiles are catalogued only", catalogOnlyProviderType) +} + +func catalogOnlyProfile(name string) query.Profile { + query.RegisterProvider(catalogOnlyProvider{}) + return query.Profile{Name: name, Provider: query.ProviderConfig{Type: catalogOnlyProviderType}} +} + +// A provider that reports it does not run is catalogued so its profiles can be +// listed, but advertising a run would hand a generic surface an entry point +// that can only fail, so it gets none — neither a run nor a session start, +// since it is not a trace the engine can capture either. +func TestProfileOpenAPICatalogOnlyProviderOffersNoOperation(t *testing.T) { spec := &rpc.OpenAPISpec{Paths: map[string]rpc.OpenAPIPath{}, Clicky: &rpc.ClickySpecMeta{}} - if err := addProfileToSpec(spec, profile); err != nil { + if err := addProfileToSpec(spec, catalogOnlyProfile("sql")); err != nil { t.Fatal(err) } const path = "/api/v1/profile/profile-sql" if _, listed := spec.Paths[path]; listed { - t.Fatalf("a legacy trace profile must advertise no run operation, got %+v", spec.Paths[path]) + t.Fatalf("a catalog-only profile must advertise no run operation, got %+v", spec.Paths[path]) } if _, listed := spec.Paths[path+"/sessions"]; listed { - t.Fatalf("a legacy trace profile must advertise no session start, got %+v", spec.Paths[path+"/sessions"]) + t.Fatalf("a catalog-only profile must advertise no session start, got %+v", spec.Paths[path+"/sessions"]) + } +} + +// A query profile whose provider is not registered cannot be judged runnable, +// so building its surface fails instead of advertising a run that cannot work. +func TestProfileOpenAPIUnregisteredProviderFails(t *testing.T) { + spec := &rpc.OpenAPISpec{Paths: map[string]rpc.OpenAPIPath{}, Clicky: &rpc.ClickySpecMeta{}} + err := addProfileToSpec(spec, query.Profile{Name: "orphan", Provider: query.ProviderConfig{Type: "unregistered-provider"}}) + if err == nil || !strings.Contains(err.Error(), `no data provider registered for type "unregistered-provider"`) { + t.Fatalf("addProfileToSpec error = %v, want the unregistered provider named", err) + } +} + +// A profile the removed legacy trace migration stored keeps its retired +// provider type, which is no longer registered; it is listed without a run +// instead of failing the whole document. +func TestProfileOpenAPIRetiredLegacyTraceProfileIsListedWithoutRun(t *testing.T) { + spec := &rpc.OpenAPISpec{Paths: map[string]rpc.OpenAPIPath{}, Clicky: &rpc.ClickySpecMeta{}} + legacy := query.Profile{Name: "legacy-sql", Provider: query.ProviderConfig{Type: retiredLegacyTraceProvider, Options: map[string]any{"kind": "sql_xevent"}}} + if err := addProfileToSpec(spec, legacy); err != nil { + t.Fatal(err) + } + const path = "/api/v1/profile/profile-legacy-sql" + if _, listed := spec.Paths[path]; listed { + t.Fatalf("a retired legacy trace profile must advertise no run operation, got %+v", spec.Paths[path]) + } + if !slices.ContainsFunc(spec.Clicky.Surfaces, func(surface rpc.ClickySurface) bool { return surface.Key == "profile-legacy-sql" }) { + t.Fatalf("a retired legacy trace profile must stay listed, got %+v", spec.Clicky.Surfaces) } } diff --git a/cmd/query/profiles/param_types_openapi_test.go b/cmd/query/profiles/param_types_openapi_test.go index 5a98c268..0afc531b 100644 --- a/cmd/query/profiles/param_types_openapi_test.go +++ b/cmd/query/profiles/param_types_openapi_test.go @@ -10,7 +10,8 @@ import ( func TestProfileOpenAPIAdvertisesDateTimeAndDurationParameterSchemas(t *testing.T) { spec := &rpc.OpenAPISpec{Paths: map[string]rpc.OpenAPIPath{}, Clicky: &rpc.ClickySpecMeta{}} if err := addProfileToSpec(spec, query.Profile{ - Name: "temporal", + Name: "temporal", + Provider: query.ProviderConfig{Type: "postgres"}, Params: []query.ParamDef{ {Name: "day", Type: query.ParamTypeDate}, {Name: "started_at", Type: query.ParamTypeDateTime}, diff --git a/cmd/query/profiles/store.go b/cmd/query/profiles/store.go index 39b90548..f766508f 100644 --- a/cmd/query/profiles/store.go +++ b/cmd/query/profiles/store.go @@ -3,7 +3,6 @@ package profiles import ( "context" "encoding/json" - "errors" "fmt" "os" "path/filepath" @@ -45,11 +44,7 @@ func NewFileStore(dir string) (*FileStore, error) { } // The dir is created by the first Save, so a store that is only read, as // to print help, writes nothing. - store := &FileStore{Dir: abs} - if err := store.migrateLegacyTraceProfiles(); err != nil { - return nil, err - } - return store, nil + return &FileStore{Dir: abs}, nil } func NewDBStore(db *gorm.DB) (*DBStore, error) { @@ -71,24 +66,7 @@ func Import(ctx context.Context, source profilestore.Store, target *DBStore) err return err } for _, profile := range items { - update := false - var existing profileRecord - err := target.db.WithContext(ctx).Where("name = ?", profile.Name).First(&existing).Error - if err == nil { - stored, decodeErr := existing.profile() - if decodeErr != nil { - return decodeErr - } - if stored.Provider.Type == legacyTraceProvider && profile.Provider.Type != legacyTraceProvider { - if stored.Provider.Connection != "" { - profile.Provider.Connection = stored.Provider.Connection - } - update = true - } - } else if !errors.Is(err, gorm.ErrRecordNotFound) { - return fmt.Errorf("inspect imported profile %q: %w", profile.Name, err) - } - if err := target.save(ctx, profile, update); err != nil { + if err := target.save(ctx, profile, false); err != nil { return fmt.Errorf("import YAML profile %q: %w", profile.Name, err) } } diff --git a/cmd/query/profiles/store_test.go b/cmd/query/profiles/store_test.go index 9aa45181..5bb55b1f 100644 --- a/cmd/query/profiles/store_test.go +++ b/cmd/query/profiles/store_test.go @@ -117,150 +117,6 @@ func TestProfileStoreListAndDelete(t *testing.T) { } } -func TestProfileStoreMigratesLegacyTraceProfilesOnStartup(t *testing.T) { - store, legacy := migratedActivityProfileStore(t) - list, err := store.List(context.Background()) - if err != nil { - t.Fatalf("List: %v", err) - } - if got := names(list); len(got) != 2 || got[0] != "Current Profile" || got[1] != "activity.client" { - t.Fatalf("expected migrated and current query profiles, got %v", got) - } - migrated := list[1] - if migrated.Provider.Type != "legacy-trace" || migrated.Provider.Options["kind"] != "sql" { - t.Fatalf("unexpected migrated provider: %+v", migrated.Provider) - } - if source, ok := migrated.Provider.Options["source"].(string); !ok || source != string(legacy) { - t.Fatalf("legacy source was not preserved: %#v", migrated.Provider.Options["source"]) - } - if len(migrated.Params) != 1 || migrated.Params[0].Name != "activity" || !migrated.Params[0].Required { - t.Fatalf("legacy params were not converted: %+v", migrated.Params) - } - if len(migrated.Columns) != 1 || migrated.Columns[0].Name != "ClientGUID" || migrated.Columns[0].CEL != "span.ClientGUID" { - t.Fatalf("legacy columns were not converted: %+v", migrated.Columns) - } -} - -func TestProfileStoreUpgradesInterimTraceImportsToExecutableProfiles(t *testing.T) { - dir := t.TempDir() - jaeger := []byte(`profile: jaeger -provider: - type: legacy-trace - options: - kind: opensearch - source: | - name: jaeger - format: jaeger - index: jaeger-span* - params: - namespace: - field: process.serviceName - operator: term -`) - jms := []byte(`profile: jms -provider: - type: legacy-trace - options: - kind: import - source: | - name: jms - imports: [jaeger] - params: - namespace: - field: process.serviceName - template: "{value}-api" - aliases: - request.xml: - cel: span["input.xml"] - request.copy: - cel: request.xml - ignore: [input.xml] -`) - for name, data := range map[string][]byte{"jaeger.yaml": jaeger, "jms.yaml": jms} { - if err := os.WriteFile(filepath.Join(dir, name), data, 0o600); err != nil { - t.Fatal(err) - } - } - store, err := NewFileStore(dir) - if err != nil { - t.Fatal(err) - } - resolved, err := Resolve(context.Background(), store, "jms") - if err != nil { - t.Fatal(err) - } - if resolved.ConnectionProfile != "jaeger" || resolved.Profile.Provider.Type != "opentelemetry" { - t.Fatalf("unexpected resolved profile: %+v", resolved) - } - if resolved.Profile.Provider.Options["index"] != "jaeger-span*" { - t.Fatalf("opentelemetry options not inherited: %#v", resolved.Profile.Provider.Options) - } - if len(resolved.Profile.Aliases) != 2 || resolved.Profile.Aliases[0].Name != "request.xml" || resolved.Profile.Aliases[1].Name != "request.copy" { - t.Fatalf("alias order not preserved: %+v", resolved.Profile.Aliases) - } -} - -func TestProfileStoreLegacyMigrationRewritesOnce(t *testing.T) { - store, _ := migratedActivityProfileStore(t) - path := filepath.Join(store.Dir, "activity.client.yaml") - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read migrated profile: %v", err) - } - var identity struct { - Profile string `json:"profile"` - Name string `json:"name"` - } - if err := yaml.Unmarshal(data, &identity); err != nil { - t.Fatalf("parse migrated profile: %v", err) - } - if identity.Profile != "activity.client" || identity.Name != "" { - t.Fatalf("profile was not rewritten to the current identity: %+v", identity) - } - if _, err := NewFileStore(store.Dir); err != nil { - t.Fatalf("second restart should leave migrated profiles unchanged: %v", err) - } - after, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read profile after second restart: %v", err) - } - if string(after) != string(data) { - t.Fatal("second startup rewrote an already migrated profile") - } -} - -func migratedActivityProfileStore(t *testing.T) (*FileStore, []byte) { - t.Helper() - dir := t.TempDir() - store, err := NewFileStore(dir) - if err != nil { - t.Fatalf("NewProfileStore: %v", err) - } - if err := store.Save(context.Background(), sampleProfile("Current Profile")); err != nil { - t.Fatalf("Save: %v", err) - } - legacy := []byte(`name: activity.client -sql: - root: AsClientActivity -params: - activity: - column: AsClientActivity.ActivityGuid - operator: terms - required: true -columns: - - name: ClientGUID - field: span.ClientGUID -`) - if err := os.WriteFile(filepath.Join(dir, "activity.client.yaml"), legacy, 0o600); err != nil { - t.Fatalf("write legacy trace profile: %v", err) - } - store, err = NewFileStore(dir) - if err != nil { - t.Fatalf("restart profile store: %v", err) - } - return store, legacy -} - func TestProfileStoreListRejectsMalformedQueryProfile(t *testing.T) { dir := t.TempDir() store, err := NewFileStore(dir) diff --git a/cmd/query/www/src/connectionOtel.test.tsx b/cmd/query/www/src/connectionOtel.test.tsx index 86c6bc47..8ada13bc 100644 --- a/cmd/query/www/src/connectionOtel.test.tsx +++ b/cmd/query/www/src/connectionOtel.test.tsx @@ -23,6 +23,7 @@ it("keeps the connection and filters when opening a trace and restoring search", await vi.waitFor(() => expect(container.textContent).toContain("Back to trace search")); expect(new URLSearchParams(window.location.search).get("index")).toBe("example-*"); expect(new URLSearchParams(window.location.search).get("trace")).toBe(summary.traceId); + expect(new URLSearchParams(window.location.search).get("tab")).toBe("traces"); await act(async () => { Array.from(container.querySelectorAll("button")).find(button => button.textContent === "Back to trace search")!.click(); }); await vi.waitFor(() => expect(container.querySelector("h2")?.textContent).toBe("Traces")); expect(new URLSearchParams(window.location.search).get("minSpans")).toBe("2"); diff --git a/otel/span/layout/flattened.go b/otel/span/layout/flattened.go new file mode 100644 index 00000000..8262b180 --- /dev/null +++ b/otel/span/layout/flattened.go @@ -0,0 +1,166 @@ +package layout + +import ( + "fmt" + "maps" + "slices" + "strings" + + "github.com/flanksource/commons-db/query/esdsl" +) + +// FlattenedAttributes maps every canonical name under Prefix to one field: the +// rest of the name, its "." replaced by Separator, under FieldPrefix. Jaeger's +// tags_as_fields writes span tag http.method as tag.http@method. +type FlattenedAttributes struct { + Prefix string `json:"prefix"` + FieldPrefix string `json:"fieldPrefix"` + Separator string `json:"separator"` +} + +// flattenedAttributeFields are the tag-field objects Jaeger's tags_as_fields +// keeps span and process attributes in. +var flattenedAttributeFields = []FlattenedAttributes{ + {Prefix: "attributes.", FieldPrefix: "tag.", Separator: "@"}, + {Prefix: "resource.", FieldPrefix: "process.tag.", Separator: "@"}, +} + +// discoverFlattened finds the tag-field objects the index maps, then validates +// any flattened rule declared by the connection layout. +func discoverFlattened(options Discovery) ([]FlattenedAttributes, error) { + mapsObject := func(rule FlattenedAttributes) bool { + return options.Fields[strings.TrimSuffix(rule.FieldPrefix, ".")] == "object" + } + rules := []FlattenedAttributes{} + for _, rule := range flattenedAttributeFields { + if mapsObject(rule) { + rules = append(rules, rule) + } + } + for _, rule := range options.FlattenedOverrides { + if rule.Prefix != "attributes." && rule.Prefix != "resource." { + return nil, fmt.Errorf("flattened layout prefix %q must be %q or %q", rule.Prefix, "attributes.", "resource.") + } + if rule.Separator == "" { + return nil, fmt.Errorf("flattened layout %s requires a separator", rule.Prefix) + } + if !strings.HasSuffix(rule.FieldPrefix, ".") || !mapsObject(rule) { + return nil, fmt.Errorf("flattened layout %s requires field prefix %q to end in \".\" and name an object the index maps", rule.Prefix, rule.FieldPrefix) + } + rules = slices.DeleteFunc(rules, func(existing FlattenedAttributes) bool { return existing.Prefix == rule.Prefix }) + rules = append(rules, rule) + } + return rules, nil +} + +func (d Definition) flattenedRule(canonical string) (FlattenedAttributes, bool) { + for _, rule := range d.Flattened { + if strings.HasPrefix(canonical, rule.Prefix) && len(canonical) > len(rule.Prefix) { + return rule, true + } + } + return FlattenedAttributes{}, false +} + +func (rule FlattenedAttributes) field(canonical string) string { + return rule.FieldPrefix + strings.ReplaceAll(strings.TrimPrefix(canonical, rule.Prefix), ".", rule.Separator) +} + +func (f Field) queryField() string { + if f.Query != "" { + return f.Query + } + return f.Source +} + +// AttributeCondition is op on attribute key of a span or of its resource, in +// source fields: one clause per place the layout holds attributes. or +// resource., any of which may match. OpExists takes no values, OpTerm and +// OpPrefix one, and OpTerms one or more. It fails when neither scope resolves, +// or when a value condition would land on a field that is not an exact keyword. +func (d Definition) AttributeCondition(key string, op esdsl.Operator, values ...string) (esdsl.Condition, error) { + if err := attributeOperands(op, values); err != nil { + return esdsl.Condition{}, fmt.Errorf("attribute %q: %w", key, err) + } + should := []esdsl.Condition{} + for _, prefix := range []string{"attributes.", "resource."} { + canonical := prefix + key + for _, target := range d.attributeTargets(canonical) { + clause, err := d.targetCondition(canonical, target, op, values) + if err != nil { + return esdsl.Condition{}, err + } + clause.Occur = esdsl.OccurShould + should = append(should, clause) + } + } + if len(should) == 0 { + return esdsl.Condition{}, fmt.Errorf("attribute %q has no source field, nested list or flattened rule in this span layout", key) + } + return esdsl.Condition{Op: esdsl.OpBool, Conditions: should}, nil +} + +func attributeOperands(op esdsl.Operator, values []string) error { + want := map[esdsl.Operator]string{esdsl.OpExists: "no values", esdsl.OpTerm: "one value", esdsl.OpPrefix: "one value", esdsl.OpTerms: "values"}[op] + switch { + case want == "": + return fmt.Errorf("operator %q has no attribute condition", op) + case op == esdsl.OpExists && len(values) > 0, (op == esdsl.OpTerm || op == esdsl.OpPrefix) && len(values) != 1, op == esdsl.OpTerms && len(values) == 0: + return fmt.Errorf("operator %q takes %s, not %d", op, want, len(values)) + } + return nil +} + +func (d Definition) targetCondition(canonical string, target Target, op esdsl.Operator, values []string) (esdsl.Condition, error) { + if err := esdsl.ValidateFieldName(target.Field); err != nil { + return esdsl.Condition{}, fmt.Errorf("%s: %w", canonical, err) + } + // A flattened rule with its own separator names a field Discover keys under + // a different canonical name, so the mapping is found by the field itself. + field, mapped := d.Fields[canonical] + if !mapped && target.Nested == "" { + field, mapped = d.fieldBySource(target.Field) + } + if mapped && op != esdsl.OpExists && field.Type != "keyword" && field.Query == "" { + return esdsl.Condition{}, fmt.Errorf("%s is mapped as %s; only a keyword field takes an exact %s condition", canonical, field.Type, op) + } + if target.Nested == "" { + // A text field is matched exactly through its keyword subfield. + if mapped { + target.Field = field.queryField() + } + return valueCondition(op, target.Field, values), nil + } + conditions := []esdsl.Condition{} + for _, key := range slices.Sorted(maps.Keys(target.Where)) { + conditions = append(conditions, esdsl.Condition{Op: esdsl.OpTerm, Field: key, Value: esdsl.Literal(target.Where[key])}) + } + if op != esdsl.OpExists { + conditions = append(conditions, valueCondition(op, target.Field, values)) + } + return esdsl.Condition{Op: esdsl.OpNested, Path: target.Nested, Conditions: conditions}, nil +} + +func (d Definition) fieldBySource(name string) (Field, bool) { + for _, field := range d.Fields { + if field.Source == name || field.Query == name { + return field, true + } + } + return Field{}, false +} + +func valueCondition(op esdsl.Operator, field string, values []string) esdsl.Condition { + switch op { + case esdsl.OpExists: + return esdsl.Condition{Op: op, Field: field} + case esdsl.OpTerms: + literals := make([]esdsl.Value, len(values)) + for i, value := range values { + literals[i] = *esdsl.Literal(value) + } + return esdsl.Condition{Op: op, Field: field, Values: literals} + default: + return esdsl.Condition{Op: op, Field: field, Value: esdsl.Literal(values[0])} + } +} diff --git a/otel/span/layout/flattened_test.go b/otel/span/layout/flattened_test.go new file mode 100644 index 00000000..70eed349 --- /dev/null +++ b/otel/span/layout/flattened_test.go @@ -0,0 +1,149 @@ +package layout_test + +import ( + "github.com/flanksource/commons-db/otel/span/layout" + "github.com/flanksource/commons-db/query/esdsl" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// jaegerFlattenedFields is a Jaeger span index written with tags_as_fields: +// span and process tags are tag. / process.tag. keyword fields +// under plain objects, with "." in a key replaced by "@". +func jaegerFlattenedFields(extra map[string]string) map[string]string { + fields := map[string]string{"traceID": "keyword", "spanID": "keyword", "startTimeMillis": "long", "duration": "long", "process": "object", "process.serviceName": "keyword", "operationName": "keyword", "tag": "object", "process.tag": "object"} + for name, typ := range extra { + fields[name] = typ + } + return fields +} + +var ( + spanTagFields = layout.FlattenedAttributes{Prefix: "attributes.", FieldPrefix: "tag.", Separator: "@"} + processTagFields = layout.FlattenedAttributes{Prefix: "resource.", FieldPrefix: "process.tag.", Separator: "@"} +) + +func compiledCondition(condition esdsl.Condition) string { + compiled, err := esdsl.Compile(esdsl.CompileRequest{Search: esdsl.Search{Query: &condition}}) + Expect(err).NotTo(HaveOccurred()) + encoded, err := compiled.JSON() + Expect(err).NotTo(HaveOccurred()) + return string(encoded) +} + +var _ = Describe("Flattened tag-field attributes", func() { + It("discovers tag and process.tag objects as flattened attribute rules and resolves unmapped keys through them", func() { + definition, err := layout.Discover(layout.Discovery{Fields: jaegerFlattenedFields(nil)}) + Expect(err).NotTo(HaveOccurred()) + Expect(definition.Flattened).To(ConsistOf(spanTagFields, processTagFields)) + Expect(definition.Nested).To(BeEmpty()) + targets, err := definition.SourceTargets("attributes.oipa.plan.guid") + Expect(err).NotTo(HaveOccurred()) + Expect(targets).To(Equal([]layout.Target{{Field: "tag.oipa@plan@guid"}})) + field, err := definition.SourceField("resource.host.name") + Expect(err).NotTo(HaveOccurred()) + Expect(field).To(Equal("process.tag.host@name")) + }) + + It("keeps a nested-only index free of flattened rules", func() { + definition, err := layout.Discover(jaegerNestedDiscovery(nil)) + Expect(err).NotTo(HaveOccurred()) + Expect(definition.Flattened).To(BeEmpty()) + }) + + It("resolves an index holding both shapes to the flattened field and the nested entry", func() { + discovery := jaegerNestedDiscovery(nil) + discovery.Fields = jaegerFlattenedFields(discovery.Fields) + definition, err := layout.Discover(discovery) + Expect(err).NotTo(HaveOccurred()) + targets, err := definition.SourceTargets("attributes.db.name") + Expect(err).NotTo(HaveOccurred()) + Expect(targets).To(Equal([]layout.Target{{Field: "tag.db@name"}, {Field: "tags.value", Nested: "tags", Where: map[string]string{"tags.key": "db.name"}}})) + }) + + It("applies a connection override to the flattened rule of its prefix", func() { + custom := layout.FlattenedAttributes{Prefix: "attributes.", FieldPrefix: "tag.", Separator: "_"} + definition, err := layout.Discover(layout.Discovery{Fields: jaegerFlattenedFields(nil), FlattenedOverrides: []layout.FlattenedAttributes{custom}}) + Expect(err).NotTo(HaveOccurred()) + Expect(definition.Flattened).To(ConsistOf(custom, processTagFields)) + field, err := definition.SourceField("attributes.http.method") + Expect(err).NotTo(HaveOccurred()) + Expect(field).To(Equal("tag.http_method")) + }) + + DescribeTable("rejects a flattened override that does not fit the index", + func(override layout.FlattenedAttributes, message string) { + _, err := layout.Discover(layout.Discovery{Fields: jaegerFlattenedFields(nil), FlattenedOverrides: []layout.FlattenedAttributes{override}}) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("an unknown prefix", layout.FlattenedAttributes{Prefix: "labels.", FieldPrefix: "tag.", Separator: "@"}, "prefix"), + Entry("a field prefix the index does not map", layout.FlattenedAttributes{Prefix: "attributes.", FieldPrefix: "span.tag.", Separator: "@"}, "span.tag."), + Entry("no separator", layout.FlattenedAttributes{Prefix: "attributes.", FieldPrefix: "tag."}, "separator"), + ) + + Describe("AttributeCondition", func() { + const plan = "oipa.plan.guid" + nested := func(path, inner string) string { + return `{"nested":{"path":"` + path + `","query":{"bool":{"filter":[{"term":{"` + path + `.key":"` + plan + `"}}` + inner + `]}}}}` + } + + It("matches a span or resource attribute through the nested lists", func() { + definition, err := layout.Discover(jaegerNestedDiscovery(nil)) + Expect(err).NotTo(HaveOccurred()) + condition, err := definition.AttributeCondition(plan, esdsl.OpTerms, "A", "B") + Expect(err).NotTo(HaveOccurred()) + Expect(compiledCondition(condition)).To(MatchJSON(`{"query":{"bool":{"minimum_should_match":1,"should":[` + + nested("tags", `,{"terms":{"tags.value":["A","B"]}}`) + `,` + nested("process.tags", `,{"terms":{"process.tags.value":["A","B"]}}`) + `]}}}`)) + }) + + It("matches a span or resource attribute through the flattened tag fields", func() { + definition, err := layout.Discover(layout.Discovery{Fields: jaegerFlattenedFields(nil)}) + Expect(err).NotTo(HaveOccurred()) + condition, err := definition.AttributeCondition(plan, esdsl.OpPrefix, "AB") + Expect(err).NotTo(HaveOccurred()) + Expect(compiledCondition(condition)).To(MatchJSON(`{"query":{"bool":{"minimum_should_match":1,"should":[{"prefix":{"tag.oipa@plan@guid":"AB"}},{"prefix":{"process.tag.oipa@plan@guid":"AB"}}]}}}`)) + }) + + It("refuses an exact condition on an attribute mapped as a non-keyword field", func() { + definition, err := layout.Discover(layout.Discovery{Fields: jaegerFlattenedFields(map[string]string{"tag.http@status_code": "long"})}) + Expect(err).NotTo(HaveOccurred()) + _, err = definition.AttributeCondition("http.status_code", esdsl.OpPrefix, "5") + Expect(err).To(MatchError(And(ContainSubstring("attributes.http.status_code"), ContainSubstring("long")))) + }) + + // Discover keys tag.http_status_code as attributes.http_status_code, since + // it only undoes "@"; the custom separator rule still lands on that field. + It("checks the mapped type of the field a custom-separator rule names", func() { + custom := layout.FlattenedAttributes{Prefix: "attributes.", FieldPrefix: "tag.", Separator: "_"} + definition, err := layout.Discover(layout.Discovery{ + Fields: jaegerFlattenedFields(map[string]string{"tag.http_status_code": "long", "tag.http_method": "keyword"}), + FlattenedOverrides: []layout.FlattenedAttributes{custom}, + }) + Expect(err).NotTo(HaveOccurred()) + _, err = definition.AttributeCondition("http.status_code", esdsl.OpPrefix, "5") + Expect(err).To(MatchError(And(ContainSubstring("attributes.http.status_code"), ContainSubstring("long")))) + condition, err := definition.AttributeCondition("http.method", esdsl.OpTerm, "GET") + Expect(err).NotTo(HaveOccurred()) + Expect(compiledCondition(condition)).To(MatchJSON(`{"query":{"bool":{"minimum_should_match":1,"should":[{"term":{"tag.http_method":"GET"}},{"term":{"process.tag.http@method":"GET"}}]}}}`)) + }) + + It("matches a text field a custom-separator rule names through its keyword subfield", func() { + custom := layout.FlattenedAttributes{Prefix: "attributes.", FieldPrefix: "tag.", Separator: "_"} + definition, err := layout.Discover(layout.Discovery{ + Fields: jaegerFlattenedFields(map[string]string{"tag.http_method": "text", "tag.http_method.keyword": "keyword"}), + FlattenedOverrides: []layout.FlattenedAttributes{custom}, + }) + Expect(err).NotTo(HaveOccurred()) + condition, err := definition.AttributeCondition("http.method", esdsl.OpTerm, "GET") + Expect(err).NotTo(HaveOccurred()) + Expect(compiledCondition(condition)).To(MatchJSON(`{"query":{"bool":{"minimum_should_match":1,"should":[{"term":{"tag.http_method.keyword":"GET"}},{"term":{"process.tag.http@method":"GET"}}]}}}`)) + }) + + It("refuses an attribute no source of the layout holds", func() { + definition, err := layout.Discover(layout.Discovery{Fields: map[string]string{"traceID": "keyword", "spanID": "keyword", "startTimeMillis": "long", "duration": "long"}}) + Expect(err).NotTo(HaveOccurred()) + _, err = definition.AttributeCondition(plan, esdsl.OpExists) + Expect(err).To(MatchError(ContainSubstring(plan))) + }) + }) +}) diff --git a/otel/span/layout/layout.go b/otel/span/layout/layout.go index 74991182..779f22c7 100644 --- a/otel/span/layout/layout.go +++ b/otel/span/layout/layout.go @@ -28,6 +28,9 @@ type Definition struct { // Nested resolves canonical attributes no scalar field maps, from a key/value // list such as Jaeger's nested `tags`, whose entries are never flattened. Nested []NestedAttributes `json:"nested,omitempty"` + // Flattened resolves canonical attributes no scalar field maps to one field + // per key, such as the tag. fields Jaeger's tags_as_fields writes. + Flattened []FlattenedAttributes `json:"flattened,omitempty"` } // NestedAttributes maps every canonical name under Prefix to the Value of the @@ -56,9 +59,10 @@ type NestedLeaf struct { type Discovery struct { Fields map[string]string // Nested holds the leaves of nested containers, keyed by full field name. - Nested map[string]NestedLeaf - Overrides map[string]Field - NestedOverrides []NestedAttributes + Nested map[string]NestedLeaf + Overrides map[string]Field + NestedOverrides []NestedAttributes + FlattenedOverrides []FlattenedAttributes } // nestedAttributeLists are the key/value lists span stores keep attributes in. @@ -210,6 +214,9 @@ func Discover(options Discovery) (Definition, error) { return d, err } d.Nested = nested + if d.Flattened, err = discoverFlattened(options); err != nil { + return d, err + } return d, nil } @@ -248,11 +255,7 @@ func (d Definition) resolve(name string, direction Direction) (string, Field, er if !ok { return "", Field{}, fmt.Errorf("canonical field %q has no source mapping", name) } - target := field.Source - if field.Query != "" { - target = field.Query - } - return target, field, nil + return field.queryField(), field, nil } if direction != ToCanonical { return "", Field{}, fmt.Errorf("invalid translation direction %q", direction) @@ -279,27 +282,64 @@ func (d Definition) resolve(name string, direction Direction) (string, Field, er // SourceField is the scalar source field of canonical. An attribute only a // nested list holds has none; read it through SourceTarget. func (d Definition) SourceField(canonical string) (string, error) { - if _, ok := d.Fields[canonical]; !ok { - if list, ok := d.nestedList(canonical); ok { - return "", fmt.Errorf("canonical field %q is held in nested %s; resolve it with SourceTarget", canonical, list.Path) - } + target, err := d.SourceTarget(canonical) + if err != nil { + return "", err } - name, _, err := d.resolve(canonical, ToSource) - return name, err + if target.Nested != "" { + return "", fmt.Errorf("canonical field %q is held in nested %s; resolve it with SourceTarget", canonical, target.Nested) + } + return target.Field, nil } // SourceTarget is where canonical is read: its scalar field when the index maps -// one, else the value of the nested list entry keyed by the attribute name. +// one, else the value of the nested list entry keyed by the attribute name, +// else the field a flattened rule names for it. func (d Definition) SourceTarget(canonical string) (Target, error) { if _, ok := d.Fields[canonical]; !ok { if list, ok := d.nestedList(canonical); ok { - return Target{Field: list.Value, Nested: list.Path, Where: map[string]string{list.Key: strings.TrimPrefix(canonical, list.Prefix)}}, nil + return list.target(canonical), nil + } + if rule, ok := d.flattenedRule(canonical); ok { + return Target{Field: rule.field(canonical)}, nil } } name, _, err := d.resolve(canonical, ToSource) return Target{Field: name}, err } +// SourceTargets is every place canonical is read, any of which may hold it: the +// scalar field the index maps, or else the field a flattened rule names, and +// the nested list entry keyed by the attribute name. An index holding both +// Jaeger tag shapes resolves an attribute to both. +func (d Definition) SourceTargets(canonical string) ([]Target, error) { + if targets := d.attributeTargets(canonical); len(targets) > 0 { + return targets, nil + } + name, _, err := d.resolve(canonical, ToSource) + if err != nil { + return nil, err + } + return []Target{{Field: name}}, nil +} + +func (d Definition) attributeTargets(canonical string) []Target { + targets := []Target{} + if field, ok := d.Fields[canonical]; ok { + targets = append(targets, Target{Field: field.queryField()}) + } else if rule, ok := d.flattenedRule(canonical); ok { + targets = append(targets, Target{Field: rule.field(canonical)}) + } + if list, ok := d.nestedList(canonical); ok { + targets = append(targets, list.target(canonical)) + } + return targets +} + +func (list NestedAttributes) target(canonical string) Target { + return Target{Field: list.Value, Nested: list.Path, Where: map[string]string{list.Key: strings.TrimPrefix(canonical, list.Prefix)}} +} + // nestedList is the nested attribute list holding canonical, by longest prefix. func (d Definition) nestedList(canonical string) (NestedAttributes, bool) { var found NestedAttributes diff --git a/otel/span/layout/search.go b/otel/span/layout/search.go index 466d9b06..8d48ea0a 100644 --- a/otel/span/layout/search.go +++ b/otel/span/layout/search.go @@ -43,7 +43,7 @@ func (d Definition) TranslateSearch(search esdsl.Search, direction Direction) (e if direction == ToCanonical { result.TimeFieldFormat = "" } else { - result.TimeFieldFormat = map[string]esdsl.TimeFieldFormat{"s": esdsl.TimeFieldFormatEpochSecond, "ms": esdsl.TimeFieldFormatEpochMillis, "us": esdsl.TimeFieldFormatEpochMicros, "ns": esdsl.TimeFieldFormatEpochNanos}[field.Unit] + result.TimeFieldFormat = epochFormats[field.Unit] } } if result.Source != nil && (len(result.Source.Includes) > 0 || len(result.Source.Excludes) > 0) || len(result.Fields) > 0 || len(result.StoredFields) > 0 || len(result.Aggregations) > 0 { diff --git a/otel/span/layout/value.go b/otel/span/layout/value.go index 7918c3fe..25b111ee 100644 --- a/otel/span/layout/value.go +++ b/otel/span/layout/value.go @@ -12,6 +12,19 @@ import ( "github.com/flanksource/commons-db/query/esdsl" ) +// epochFormats are the search time formats of the numeric source units. +var epochFormats = map[string]esdsl.TimeFieldFormat{"s": esdsl.TimeFieldFormatEpochSecond, "ms": esdsl.TimeFieldFormatEpochMillis, "us": esdsl.TimeFieldFormatEpochMicros, "ns": esdsl.TimeFieldFormatEpochNanos} + +// TimeFieldFormat is the epoch format a search bounding f declares: that of +// its Unit for a numeric field, and none for a date field, whose mapping +// already says how it stores instants. +func (f Field) TimeFieldFormat() esdsl.TimeFieldFormat { + if f.Type == "date" || f.Type == "date_nanos" { + return "" + } + return epochFormats[f.Unit] +} + func (f Field) validateOrdering() error { if len(f.Values) > 0 { return fmt.Errorf("source enum %s has no reversible canonical ordering", f.Source) @@ -53,7 +66,7 @@ func (f Field) value(value any, canonical string, direction Direction) (any, err } if canonical == "timestamp" && f.Unit != "" { if direction == ToSource { - format := map[string]esdsl.TimeFieldFormat{"s": esdsl.TimeFieldFormatEpochSecond, "ms": esdsl.TimeFieldFormatEpochMillis, "us": esdsl.TimeFieldFormatEpochMicros, "ns": esdsl.TimeFieldFormatEpochNanos}[f.Unit] + format := epochFormats[f.Unit] if format == "" { return nil, fmt.Errorf("unknown timestamp unit %q", f.Unit) } diff --git a/query/engine_test.go b/query/engine_test.go index 9e064eed..c2677c45 100644 --- a/query/engine_test.go +++ b/query/engine_test.go @@ -36,6 +36,13 @@ func (p *capabilityProvider) BackendCapabilities( return p.capabilities, p.err } +type runnableProvider struct { + *mockProvider + runnable bool +} + +func (p *runnableProvider) Runnable() bool { return p.runnable } + func (m *mockProvider) Type() string { return m.typ } func (m *mockProvider) Execute(ctx context.Context, req query.ProviderRequest) ([]query.Row, error) { @@ -99,6 +106,23 @@ var _ = Describe("provider registry", func() { Expect(err).To(MatchError(ContainSubstring("metadata denied"))) }) + DescribeTable("reports whether a provider type runs", + func(provider query.Provider, want bool) { + query.RegisterProvider(provider) + runnable, err := query.ProviderRunnable(provider.Type()) + Expect(err).ToNot(HaveOccurred()) + Expect(runnable).To(Equal(want)) + }, + Entry("a provider without the optional interface runs", &mockProvider{typ: "runnable-implicit"}, true), + Entry("a provider that reports it runs", &runnableProvider{mockProvider: &mockProvider{typ: "runnable-explicit"}, runnable: true}, true), + Entry("a catalog-only provider does not run", &runnableProvider{mockProvider: &mockProvider{typ: "runnable-catalog-only"}}, false), + ) + + It("fails to judge an unregistered provider type as runnable", func() { + _, err := query.ProviderRunnable("runnable-unregistered") + Expect(err).To(MatchError(ContainSubstring(`no data provider registered for type "runnable-unregistered"`))) + }) + It("supports concurrent registration, lookup, and enumeration", func() { const workers = 64 start := make(chan struct{}) diff --git a/query/provider.go b/query/provider.go index 7c101bc2..68e8f3b6 100644 --- a/query/provider.go +++ b/query/provider.go @@ -64,6 +64,24 @@ func RequireBackendCapability( return capabilities, nil } +// RunnableProvider is an optional Provider capability. A provider reporting +// false is registered so its profiles can be catalogued, but it refuses every +// execution, so surfaces offer no run operation for it. +type RunnableProvider interface { + Runnable() bool +} + +// ProviderRunnable reports whether the provider registered for typ executes +// profiles. A provider without RunnableProvider runs. +func ProviderRunnable(typ string) (bool, error) { + provider, err := GetProvider(typ) + if err != nil { + return false, err + } + runnable, ok := provider.(RunnableProvider) + return !ok || runnable.Runnable(), nil +} + // QueryParameterizer replaces provider-native parameter references with opaque // markers while carrying their values separately to the backend driver. type QueryParameterizer interface { diff --git a/query/providers/canonical_search.go b/query/providers/canonical_search.go index ab4aa820..443dffd9 100644 --- a/query/providers/canonical_search.go +++ b/query/providers/canonical_search.go @@ -21,85 +21,122 @@ func resolveSpanLayout(ctx context.Context, searcher *opensearch.Searcher, index if err := req.Spans.ValidateReversible(); err != nil { return nil, err } - if index == "" { - return nil, fmt.Errorf("canonical search requires an index on the profile or connection") - } - inspector, err := opensearchinspect.New(searcher.GetRawClient(), opensearchinspect.Options{CacheKey: searcher.InspectionKey()}) + overrides, err := decodeSpanLayout(configured) if err != nil { return nil, err } - catalog, err := inspector.Fields(ctx, opensearchinspect.FieldRequest{Target: openSearchTimeTarget(index), Refresh: req.Inspection.Refresh}) - if err != nil { - return nil, fmt.Errorf("resolve span layout on %s: %w", index, err) - } - if catalog.Truncated { - return nil, fmt.Errorf("span layout inspection is incomplete: %s", catalog.TruncateReason) - } - fields := map[string]string{} - // Nested leaves are kept apart: a key/value attribute list is only queryable - // through a nested clause, never as a scalar field. - nested := map[string]layout.NestedLeaf{} - for _, field := range catalog.Fields { - if field.Nested() { - if !field.Conflicting && len(field.Types) == 1 { - nested[field.Name] = layout.NestedLeaf{Type: field.Types[0], Container: field.Container} - } - continue - } - if field.Conflicting || len(field.Types) != 1 { - return nil, fmt.Errorf("span layout field %s has conflicting mapping types", field.Name) + for name, mapping := range req.Spans { + canonical := map[string]string{"traceId": "trace_id", "spanId": "span_id", "parentId": "parent_id", "startedAt": "timestamp", "durationMs": "duration_ms", "service": "service", "operation": "operation", "status": "status", "kind": "kind"}[name] + if canonical == "" { + return nil, fmt.Errorf("span mapping %s has no reversible query field", name) } - fields[field.Name] = field.Types[0] + overrides.Fields[canonical] = layout.Field{Source: mapping.Source, Unit: mapping.Unit} } - overrides := layout.Definition{Fields: map[string]layout.Field{}} + resolved, err := discoverSpanLayout(ctx, spanLayoutSource{searcher: searcher, index: index, refresh: req.Inspection.Refresh}, overrides) + if err != nil { + return nil, err + } + return &resolved, nil +} + +// decodeSpanLayout strictly reads a connection's span_layout property: the +// overrides it applies over the layout discovered from the index mapping. +func decodeSpanLayout(configured string) (layout.Definition, error) { + overrides := layout.Definition{} if configured != "" { decoder := json.NewDecoder(strings.NewReader(configured)) decoder.DisallowUnknownFields() decoder.UseNumber() if err := decoder.Decode(&overrides); err != nil { - return nil, fmt.Errorf("connection span_layout: %w", err) + return overrides, fmt.Errorf("connection span_layout: %w", err) } if err := decoder.Decode(new(any)); err != io.EOF { - return nil, fmt.Errorf("connection span_layout must contain one JSON object") + return overrides, fmt.Errorf("connection span_layout must contain one JSON object") } } - for name, mapping := range req.Spans { - canonical := map[string]string{"traceId": "trace_id", "spanId": "span_id", "parentId": "parent_id", "startedAt": "timestamp", "durationMs": "duration_ms", "service": "service", "operation": "operation", "status": "status", "kind": "kind"}[name] - if canonical == "" { - return nil, fmt.Errorf("span mapping %s has no reversible query field", name) - } - if overrides.Fields == nil { - overrides.Fields = map[string]layout.Field{} - } - overrides.Fields[canonical] = layout.Field{Source: mapping.Source, Unit: mapping.Unit} + if overrides.Fields == nil { + overrides.Fields = map[string]layout.Field{} + } + return overrides, nil +} + +type spanLayoutSource struct { + searcher *opensearch.Searcher + index string + refresh bool +} + +// discoverSpanLayout resolves the layout of source's index from its mapped +// fields, read through the cached OpenSearch inspection, with overrides applied. +func discoverSpanLayout(ctx context.Context, source spanLayoutSource, overrides layout.Definition) (layout.Definition, error) { + if source.index == "" { + return layout.Definition{}, fmt.Errorf("span layout requires an index on the profile or connection") } - resolved, err := layout.Discover(layout.Discovery{Fields: fields, Nested: nested, Overrides: overrides.Fields, NestedOverrides: overrides.Nested}) + inspector, err := opensearchinspect.New(source.searcher.GetRawClient(), opensearchinspect.Options{CacheKey: source.searcher.InspectionKey()}) if err != nil { - return nil, err + return layout.Definition{}, err + } + catalog, err := inspector.Fields(ctx, opensearchinspect.FieldRequest{Target: openSearchTimeTarget(source.index), Refresh: source.refresh}) + if err != nil { + return layout.Definition{}, fmt.Errorf("resolve span layout on %s: %w", source.index, err) + } + discovery, err := spanLayoutDiscovery(catalog) + if err != nil { + return layout.Definition{}, err + } + discovery.Overrides, discovery.NestedOverrides, discovery.FlattenedOverrides = overrides.Fields, overrides.Nested, overrides.Flattened + resolved, err := layout.Discover(discovery) + if err != nil { + return layout.Definition{}, err } clock := resolved.Fields["timestamp"] if clock.Type == "date" || clock.Type == "date_nanos" { - catalog, err := inspector.Fields(ctx, opensearchinspect.FieldRequest{Target: openSearchTimeTarget(index), Names: []string{clock.Source}, IncludeFormats: true, Refresh: req.Inspection.Refresh}) - if err != nil { - return nil, fmt.Errorf("resolve timestamp format: %w", err) + if clock.Format, err = timestampFormat(ctx, inspector, source, clock.Source); err != nil { + return layout.Definition{}, err } - found := false - for _, field := range catalog.Fields { - if field.Name != clock.Source { - continue - } - if field.FormatConflicting { - return nil, fmt.Errorf("timestamp %s has conflicting date formats", clock.Source) + resolved.Fields["timestamp"] = clock + } + return resolved, nil +} + +// spanLayoutDiscovery splits the inspected fields into scalar fields and the +// leaves of nested containers: a key/value attribute list is only queryable +// through a nested clause, never as a scalar field. +func spanLayoutDiscovery(catalog opensearchinspect.FieldCatalog) (layout.Discovery, error) { + if catalog.Truncated { + return layout.Discovery{}, fmt.Errorf("span layout inspection is incomplete: %s", catalog.TruncateReason) + } + discovery := layout.Discovery{Fields: map[string]string{}, Nested: map[string]layout.NestedLeaf{}} + for _, field := range catalog.Fields { + if field.Nested() { + if !field.Conflicting && len(field.Types) == 1 { + discovery.Nested[field.Name] = layout.NestedLeaf{Type: field.Types[0], Container: field.Container} } - clock.Format = field.Format - found = true + continue } - if !found { - return nil, fmt.Errorf("timestamp %s has no resolved date format", clock.Source) + if field.Conflicting || len(field.Types) != 1 { + return layout.Discovery{}, fmt.Errorf("span layout field %s has conflicting mapping types", field.Name) } - resolved.Fields["timestamp"] = clock + discovery.Fields[field.Name] = field.Types[0] } - return &resolved, nil + return discovery, nil +} + +func timestampFormat(ctx context.Context, inspector *opensearchinspect.Inspector, source spanLayoutSource, name string) (string, error) { + catalog, err := inspector.Fields(ctx, opensearchinspect.FieldRequest{Target: openSearchTimeTarget(source.index), Names: []string{name}, IncludeFormats: true, Refresh: source.refresh}) + if err != nil { + return "", fmt.Errorf("resolve timestamp format: %w", err) + } + for _, field := range catalog.Fields { + if field.Name != name { + continue + } + if field.FormatConflicting { + return "", fmt.Errorf("timestamp %s has conflicting date formats", name) + } + return field.Format, nil + } + return "", fmt.Errorf("timestamp %s has no resolved date format", name) } func canonicalSearchRequest(req query.ProviderRequest, page openSearchPage, resolved *layout.Definition) (openSearchRequest, error) { diff --git a/query/providers/opentelemetry.go b/query/providers/opentelemetry.go index fe1a331e..b0eab01e 100644 --- a/query/providers/opentelemetry.go +++ b/query/providers/opentelemetry.go @@ -40,9 +40,15 @@ type OpenTelemetryOptions struct { Layout *layout.Definition `json:"-"` RawRows bool `json:"-"` OpenSearchPagingOptions - Format string `json:"format,omitempty"` - Index string `json:"index,omitempty"` - DateField string `json:"dateField,omitempty"` + Format string `json:"format,omitempty"` + Index string `json:"index,omitempty"` + DateField string `json:"dateField,omitempty"` + // TraceIDField, SpanIDField and ServiceField name the fields a row's span + // identity and service are normalized from, when the span layout resolves + // them to a field outside the names normalization tries. + TraceIDField string `json:"traceIdField,omitempty"` + SpanIDField string `json:"spanIdField,omitempty"` + ServiceField string `json:"serviceField,omitempty"` SelectFields []string `json:"selectFields,omitempty"` SourceExcludes []string `json:"sourceExcludes,omitempty"` Limit int `json:"limit,omitempty"` @@ -57,6 +63,8 @@ type openTelemetryRuntime struct { searcher *opensearch.Searcher options OpenTelemetryOptions paging openSearchPagingMode + // spanLayout is the connection's span_layout property, unparsed. + spanLayout string } // PagingModes matches the nested OpenSearch connection: from/size inside the @@ -188,25 +196,10 @@ func openTelemetrySearchClient(ctx context.Context, req query.ProviderRequest) ( return openTelemetryRuntime{}, fmt.Errorf( "provider.options.params has been replaced by provider.options.search; re-import the profile to migrate it") } - options.withDefaults() options.RawRows = req.RawRows || req.Search != nil - outerModel, err := ctx.HydrateConnectionByURL(req.Connection) + outer, nested, err := openTelemetryConnections(ctx, req.Connection) if err != nil { - return openTelemetryRuntime{}, fmt.Errorf("hydrate opentelemetry connection %q: %w", req.Connection, err) - } - nested := outerModel - if outerModel == nil { - return openTelemetryRuntime{}, fmt.Errorf("trace connection %q not found", req.Connection) - } - if outerModel.Type != models.ConnectionTypeOpenSearch { - outer, err := dbconnection.NewOpenTelemetry(outerModel) - if err != nil { - return openTelemetryRuntime{}, err - } - nested, err = outer.ResolveOpenSearch(ctx) - if err != nil { - return openTelemetryRuntime{}, err - } + return openTelemetryRuntime{}, err } runtime, err := openSearchClientForConnection(ctx, nested) if err != nil { @@ -216,22 +209,81 @@ func openTelemetrySearchClient(ctx context.Context, req query.ProviderRequest) ( if err != nil { return openTelemetryRuntime{}, err } + // The request's options win, then the opentelemetry connection's own + // properties, then those of the OpenSearch connection it wraps. if req.Options["index"] == nil { - if index := outerModel.Properties["index"]; index != "" { - options.Index = index - } else if index := nested.Properties["index"]; index != "" { - options.Index = index - } + options.Index = connectionProperty("index", outer, nested) } - configured := outerModel.Properties["span_layout"] - if configured == "" { - configured = nested.Properties["span_layout"] + if req.Options["format"] == nil { + options.Format = connectionProperty("format", outer, nested) + if err := options.Validate(); err != nil { + return openTelemetryRuntime{}, fmt.Errorf("connection %q: %w", req.Connection, err) + } } - options.Layout, err = resolveSpanLayout(ctx, runtime.searcher, options.Index, req, configured) + options.withDefaults() + spanLayout := connectionProperty("span_layout", outer, nested) + options.Layout, err = resolveSpanLayout(ctx, runtime.searcher, options.Index, req, spanLayout) if err != nil { return openTelemetryRuntime{}, err } - return openTelemetryRuntime{searcher: runtime.searcher, options: options, paging: policy.mode, threshold: policy.threshold}, nil + return openTelemetryRuntime{searcher: runtime.searcher, options: options, paging: policy.mode, threshold: policy.threshold, spanLayout: spanLayout}, nil +} + +// openTelemetryConnections hydrates reference and the OpenSearch connection +// holding its spans: the nested one an opentelemetry connection names, or the +// connection itself when it is OpenSearch. +func openTelemetryConnections(ctx context.Context, reference string) (outer, nested *models.Connection, err error) { + outer, err = ctx.HydrateConnectionByURL(reference) + if err != nil { + return nil, nil, fmt.Errorf("hydrate opentelemetry connection %q: %w", reference, err) + } + if outer == nil { + return nil, nil, fmt.Errorf("trace connection %q not found", reference) + } + if outer.Type == models.ConnectionTypeOpenSearch { + return outer, outer, nil + } + wrapper, err := dbconnection.NewOpenTelemetry(outer) + if err != nil { + return nil, nil, err + } + nested, err = wrapper.ResolveOpenSearch(ctx) + return outer, nested, err +} + +func connectionProperty(key string, outer, nested *models.Connection) string { + if value := outer.Properties[key]; value != "" { + return value + } + return nested.Properties[key] +} + +// SpanLayoutRequest names the index whose span layout is resolved: the one an +// opentelemetry read of Connection with Options would search. +type SpanLayoutRequest struct { + // Connection is an opentelemetry or opensearch connection reference. + Connection string + // Options are the opentelemetry provider's options; their index wins over + // the connection's. + Options map[string]any + // Refresh re-reads the index mapping instead of the cached inspection. + Refresh bool +} + +// ResolveSpanLayout discovers the span layout of a connection's index: its +// mapped fields, with the connection's span_layout applied as overrides. The +// mapping is read through the OpenSearch inspection memo, so it is cached per +// connection and index, bounded and refreshed by that memo's policy. +func ResolveSpanLayout(ctx context.Context, request SpanLayoutRequest) (layout.Definition, error) { + runtime, err := openTelemetrySearchClient(ctx, query.ProviderRequest{Connection: request.Connection, Options: request.Options}) + if err != nil { + return layout.Definition{}, err + } + overrides, err := decodeSpanLayout(runtime.spanLayout) + if err != nil { + return layout.Definition{}, err + } + return discoverSpanLayout(ctx, spanLayoutSource{searcher: runtime.searcher, index: runtime.options.Index, refresh: request.Refresh}, overrides) } // Validate refuses a format the provider cannot shape rows from; empty is flat. diff --git a/query/providers/opentelemetry_parser.go b/query/providers/opentelemetry_parser.go index c7c0b1e1..2f80cca1 100644 --- a/query/providers/opentelemetry_parser.go +++ b/query/providers/opentelemetry_parser.go @@ -9,7 +9,7 @@ func openTelemetryRow(document map[string]any, options OpenTelemetryOptions) (qu if options.RawRows { return query.Row(document), nil } - normalized, err := span.Normalize(document, span.Options{Format: options.Format, DateField: options.DateField}) + normalized, err := span.Normalize(document, span.Options{Format: options.Format, DateField: options.DateField, TraceIDField: options.TraceIDField, SpanIDField: options.SpanIDField, ServiceField: options.ServiceField}) if err != nil { return nil, err } diff --git a/query/providers/opentelemetry_read_test.go b/query/providers/opentelemetry_read_test.go index 9e42c359..822707a8 100644 --- a/query/providers/opentelemetry_read_test.go +++ b/query/providers/opentelemetry_read_test.go @@ -105,6 +105,35 @@ var _ = Describe("ReadOpenTelemetrySpans", func() { Eventually(done).Should(Receive(BeNil())) }) + DescribeTable("reads the document format from the request, then the connection, then its nested connection", + func(requested string, outer, nested types.JSONStringMap, rejected bool) { + ctx = ctx.WithConnectionResolver(func(_ dbcontext.Context, reference string) (*models.Connection, error) { + switch reference { + case "connection://traces": + outer["connection"] = "connection://spans" + return &models.Connection{Name: "traces", Type: models.ConnectionTypeOpenTelemetry, Properties: outer}, nil + case "connection://spans": + return &models.Connection{Name: "spans", Type: models.ConnectionTypeOpenSearch, URL: stub.server.URL, Properties: nested}, nil + } + return nil, nil + }) + options := map[string]any{"index": "logs"} + if requested != "" { + options["format"] = requested + } + err := providers.ReadOpenTelemetrySpans(ctx, providers.OpenTelemetryRead{Connection: "connection://traces", Options: options}, func(query.Row) {}) + if rejected { + Expect(err).To(MatchError(ContainSubstring(`unsupported opentelemetry format "bogus"`))) + } else { + Expect(err).NotTo(HaveOccurred()) + } + }, + Entry("the connection's format", "", types.JSONStringMap{"format": "bogus"}, nil, true), + Entry("the nested connection's format", "", types.JSONStringMap{}, types.JSONStringMap{"format": "bogus"}, true), + Entry("the connection over its nested connection", "", types.JSONStringMap{"format": "flat"}, types.JSONStringMap{"format": "bogus"}, false), + Entry("the request over the connection", "flat", types.JSONStringMap{"format": "bogus"}, nil, false), + ) + It("refuses a followed read with an end", func() { err := providers.ReadOpenTelemetrySpans(ctx, providers.OpenTelemetryRead{ Connection: "connection://traces", Follow: true, To: "now", diff --git a/query/providers/span_layout_test.go b/query/providers/span_layout_test.go new file mode 100644 index 00000000..f3e45170 --- /dev/null +++ b/query/providers/span_layout_test.go @@ -0,0 +1,67 @@ +package providers_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/models" + "github.com/flanksource/commons-db/otel/span/layout" + "github.com/flanksource/commons-db/query/providers" + "github.com/flanksource/commons-db/types" +) + +// tagsAsFieldsCatalog is the _field_caps of a Jaeger index written with +// tags_as_fields. +var tagsAsFieldsCatalog = map[string]string{ + "traceID": "keyword", "spanID": "keyword", "startTimeMillis": "long", "duration": "long", + "operationName": "keyword", "process": "object", "process.serviceName": "keyword", + "tag": "object", "process.tag": "object", "tag.http@method": "keyword", +} + +var _ = Describe("ResolveSpanLayout", func() { + It("discovers a connection index's tag fields and applies the connection span_layout over them", func() { + inspected := []string{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer GinkgoRecover() + if r.Method == http.MethodHead { + return + } + Expect(r.URL.Path).To(HaveSuffix("/_field_caps")) + inspected = append(inspected, strings.TrimSuffix(r.URL.Path, "/_field_caps")) + fields := map[string]any{} + for name, typ := range tagsAsFieldsCatalog { + fields[name] = map[string]any{typ: map[string]any{"searchable": true, "aggregatable": true}} + } + w.Header().Set("Content-Type", "application/json") + Expect(json.NewEncoder(w).Encode(map[string]any{"fields": fields})).To(Succeed()) + })) + DeferCleanup(server.Close) + ctx := dbcontext.New().WithConnectionResolver(func(_ dbcontext.Context, reference string) (*models.Connection, error) { + switch reference { + case "connection://traces": + return &models.Connection{Name: "traces", Type: models.ConnectionTypeOpenTelemetry, Properties: types.JSONStringMap{ + "connection": "connection://spans", "index": "jaeger-span-*", + "span_layout": `{"flattened":[{"prefix":"resource.","fieldPrefix":"process.tag.","separator":"_"}]}`, + }}, nil + case "connection://spans": + return &models.Connection{Name: "spans", Type: models.ConnectionTypeOpenSearch, URL: server.URL}, nil + } + return nil, nil + }) + + definition, err := providers.ResolveSpanLayout(ctx, providers.SpanLayoutRequest{Connection: "connection://traces"}) + Expect(err).NotTo(HaveOccurred()) + Expect(inspected).To(Equal([]string{"/jaeger-span-*"})) + Expect(definition.Fields["timestamp"]).To(Equal(layout.Field{Source: "startTimeMillis", Type: "long", Unit: "ms"})) + Expect(definition.Flattened).To(ConsistOf( + layout.FlattenedAttributes{Prefix: "attributes.", FieldPrefix: "tag.", Separator: "@"}, + layout.FlattenedAttributes{Prefix: "resource.", FieldPrefix: "process.tag.", Separator: "_"}, + )) + }) +}) diff --git a/query/schema/connection.go b/query/schema/connection.go index b4f27f33..51811fd1 100644 --- a/query/schema/connection.go +++ b/query/schema/connection.go @@ -354,6 +354,9 @@ func tailoredBranch(typ string, cfg any) Schema { if typ == models.ConnectionTypeClickHouse { customizeClickHousePropertiesSchema(properties) } + if typ == models.ConnectionTypeOpenTelemetry { + customizeOpenTelemetryPropertiesSchema(properties) + } if len(propertyRequired) > 0 { properties["required"] = propertyRequired required = append(required, "properties") @@ -428,6 +431,14 @@ func customizeOpenSearchPropertiesSchema(properties Schema) { } } +func customizeOpenTelemetryPropertiesSchema(properties Schema) { + field := properties["properties"].(Schema)["format"].(Schema) + field["description"] = "How span documents are shaped: flat OpenTelemetry fields, or Jaeger's span documents." + field["enum"] = []string{"flat", "jaeger"} + field["x-enum-display"] = "segmented" + field["x-enum-labels"] = map[string]string{"flat": "Flat", "jaeger": "Jaeger"} +} + func customizeClickHousePropertiesSchema(properties Schema) { properties["title"] = "Query limits" properties["description"] = "Applied to every query opened through this connection. Missing values use these safe defaults." diff --git a/query/schema/connection_providers.go b/query/schema/connection_providers.go index edd090bd..e565002d 100644 --- a/query/schema/connection_providers.go +++ b/query/schema/connection_providers.go @@ -45,7 +45,8 @@ type OpenSearchProvider struct { // connection while retaining its own first-class connection identity. type OpenTelemetryProvider struct { Index string `json:"index" clicky:"property=index,title=Default index,order=3"` - SpanLayout string `json:"span_layout" clicky:"property=span_layout,title=Span layout overrides,format=textarea,order=4"` + Format string `json:"format" clicky:"property=format,title=Document format,order=4"` + SpanLayout string `json:"span_layout" clicky:"property=span_layout,title=Span layout overrides,format=textarea,order=5"` Connection string `json:"connection" clicky:"property=connection,title=OpenSearch Connection,required,order=2"` } diff --git a/query/schema/schema_test.go b/query/schema/schema_test.go index 6181a7e4..34e4d2b3 100644 --- a/query/schema/schema_test.go +++ b/query/schema/schema_test.go @@ -195,6 +195,22 @@ var _ = Describe("Connection schema", func() { Expect(scope["map"].(map[string][]string)[models.ConnectionTypeOpenTelemetry]).To(Equal([]string{models.ConnectionTypeOpenSearch})) }) + // An omitted format is auto-detected from each document, so the form must + // not save one: a default would parse Jaeger documents as flat. + It("offers OpenTelemetry a span document format with no default", func() { + then := branchFor(s, models.ConnectionTypeOpenTelemetry) + settings := then["properties"].(schema.Schema)["properties"].(schema.Schema)["properties"].(schema.Schema) + Expect(settings).To(HaveKey("index")) + Expect(settings).To(HaveKey("span_layout")) + format := settings["format"].(schema.Schema) + Expect(format).To(And( + HaveKeyWithValue("enum", []string{"flat", "jaeger"}), + Not(HaveKey("default")), + HaveKeyWithValue("x-enum-display", "segmented"), + HaveKeyWithValue("x-enum-labels", map[string]string{"flat": "Flat", "jaeger": "Jaeger"}), + )) + }) + It("gives Elasticsearch the OpenSearch endpoint form", func() { // Without a tailored branch the type falls back to name/namespace and // the connection cannot be given an endpoint at all — a picker entry diff --git a/query/tracequery/filter.go b/query/tracequery/filter.go index a5856431..604d62cb 100644 --- a/query/tracequery/filter.go +++ b/query/tracequery/filter.go @@ -64,11 +64,6 @@ func (m Modes) pass(has func(string) bool) bool { // expressions are any-of per key and all-of across keys, and any span // matching an excluded expression rejects the trace. func (m Modes) passAttributes(spans []Span) bool { - byKey := map[string][]string{} - for _, expr := range m.values(Include) { - key, _, _ := strings.Cut(expr, "=") - byKey[key] = append(byKey[key], expr) - } anySpan := func(expr string) bool { for _, span := range spans { if attributeMatches(span, expr) { @@ -77,7 +72,7 @@ func (m Modes) passAttributes(spans []Span) bool { } return false } - for _, exprs := range byKey { + for _, exprs := range m.includedByKey() { matched := false for _, expr := range exprs { matched = matched || anySpan(expr) @@ -94,6 +89,17 @@ func (m Modes) passAttributes(spans []Span) bool { return true } +// includedByKey groups the included attribute expressions by their key; a +// trace passes a group when a span matches any expression of it. +func (m Modes) includedByKey() map[string][]string { + byKey := map[string][]string{} + for _, expr := range m.values(Include) { + key, _, _ := strings.Cut(expr, "=") + byKey[key] = append(byKey[key], expr) + } + return byKey +} + func validAttribute(expr string) error { key, _, _ := strings.Cut(expr, "=") if strings.TrimSpace(key) == "" { diff --git a/query/tracequery/handler.go b/query/tracequery/handler.go new file mode 100644 index 00000000..d491db5d --- /dev/null +++ b/query/tracequery/handler.go @@ -0,0 +1,102 @@ +package tracequery + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/flanksource/clicky/rpc" +) + +const handlerTimeout = 30 * time.Second + +// Handler serves service's traces of source relative to where it is mounted +// (behind http.StripPrefix): GET / searches with the query parameters +// ParseRequest reads, and GET /{traceId} opens one trace. An Accept of +// application/schema+json returns the Request schema instead. +func Handler(service Service, source string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "traces are read with GET", http.StatusMethodNotAllowed) + return + } + if r.Header.Get("Accept") == "application/schema+json" { + writeJSON(w, rpc.SchemaForStruct(Request{})) + return + } + values, err := url.ParseQuery(r.URL.RawQuery) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + request, err := ParseRequest(values) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + request.Source = source + ctx, cancel := context.WithTimeout(r.Context(), handlerTimeout) + defer cancel() + var result any + if id := strings.Trim(r.URL.Path, "/"); id == "" { + result, err = service.Search(ctx, request) + } else { + result, err = service.Trace(ctx, request, id) + } + if err != nil { + status := http.StatusBadRequest + if errors.Is(err, ErrTraceNotFound) { + status = http.StatusNotFound + } + http.Error(w, err.Error(), status) + return + } + writeJSON(w, result) + }) +} + +// ParseRequest reads a Request from query parameters named by its JSON +// fields, each given once: integers as numbers and Modes as a JSON object. +func ParseRequest(values url.Values) (Request, error) { + schema := rpc.SchemaForStruct(Request{}) + body := map[string]any{} + for key, items := range values { + property, ok := schema.Properties[key] + if !ok || len(items) != 1 { + return Request{}, fmt.Errorf("unknown or repeated trace parameter %q", key) + } + var err error + switch property.Type { + case "integer": + body[key], err = strconv.Atoi(items[0]) + case "object": + var modes map[string]string + err = json.Unmarshal([]byte(items[0]), &modes) + body[key] = modes + default: + body[key] = items[0] + } + if err != nil { + return Request{}, fmt.Errorf("%s: %w", key, err) + } + } + data, err := json.Marshal(body) + if err != nil { + return Request{}, err + } + var request Request + err = json.Unmarshal(data, &request) + return request, err +} + +func writeJSON(w http.ResponseWriter, payload any) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Access-Control-Allow-Origin", "*") + _ = json.NewEncoder(w).Encode(payload) +} diff --git a/query/tracequery/handler_test.go b/query/tracequery/handler_test.go new file mode 100644 index 00000000..72cb3d3a --- /dev/null +++ b/query/tracequery/handler_test.go @@ -0,0 +1,70 @@ +package tracequery + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("trace HTTP handler", func() { + It("decodes typed trace filters and rejects malformed parameters", func() { + values, err := url.ParseQuery("minSpans=2&services=%7B%22checkout%22:%22include%22%7D") + Expect(err).NotTo(HaveOccurred()) + request, err := ParseRequest(values) + Expect(err).NotTo(HaveOccurred()) + Expect(request).To(Equal(Request{MinSpans: 2, Services: Modes{"checkout": Include}})) + for _, query := range []string{"minSpans=oops", "unknown=value", "limit=1&limit=2", "services=invalid"} { + values, err := url.ParseQuery(query) + Expect(err).NotTo(HaveOccurred()) + _, err = ParseRequest(values) + Expect(err).To(HaveOccurred(), query) + } + }) + + It("searches at its root and opens a trace below it, for its fixed source", func() { + at := time.Now().UTC().Add(-time.Minute).Truncate(time.Millisecond) + sources := []string{} + service := Service{Read: func(_ context.Context, r Request, _ []string) ([]Span, error) { + sources = append(sources, r.Source) + return []Span{ + {TraceID: "trace-a", SpanID: "root", StartedAt: at, DurationMs: 5, Service: "checkout", Operation: "root", Status: "ok"}, + {TraceID: "trace-a", SpanID: "child", ParentID: "root", StartedAt: at.Add(time.Millisecond), DurationMs: 1, Service: "checkout", Operation: "child", Status: "error"}, + }, nil + }} + handler := http.StripPrefix("/traces", Handler(service, "fixed")) + serve := func(target string, header http.Header) *httptest.ResponseRecorder { + request := httptest.NewRequest(http.MethodGet, target, nil) + request.Header = header + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + return response + } + + response := serve("/traces?minSpans=2&source=ignored", http.Header{}) + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + var result SearchResult + Expect(json.Unmarshal(response.Body.Bytes(), &result)).To(Succeed()) + Expect(result.Traces).To(HaveExactElements(And(HaveField("TraceID", "trace-a"), HaveField("SpanCount", 2), HaveField("Errors", 1)))) + + response = serve("/traces/trace-a", http.Header{}) + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + var detail Detail + Expect(json.Unmarshal(response.Body.Bytes(), &detail)).To(Succeed()) + Expect(detail.Spans).To(HaveLen(2)) + Expect(sources).To(HaveEach("fixed")) + + response = serve("/traces/", http.Header{"Accept": {"application/schema+json"}}) + Expect(response.Code).To(Equal(http.StatusOK)) + Expect(response.Body.String()).To(ContainSubstring(`"attributes"`)) + + Expect(serve("/traces/trace-missing", http.Header{}).Code).To(Equal(http.StatusNotFound)) + Expect(serve("/traces?minSpans=oops", http.Header{}).Code).To(Equal(http.StatusBadRequest)) + Expect(serve("/traces?lookback=-1h", http.Header{}).Code).To(Equal(http.StatusBadRequest)) + }) +}) diff --git a/query/tracequery/opensearch.go b/query/tracequery/opensearch.go index 75c376aa..b9471873 100644 --- a/query/tracequery/opensearch.go +++ b/query/tracequery/opensearch.go @@ -3,61 +3,165 @@ package tracequery import ( "context" "fmt" + "maps" + "slices" dbcontext "github.com/flanksource/commons-db/context" "github.com/flanksource/commons-db/models" + "github.com/flanksource/commons-db/otel/span/layout" "github.com/flanksource/commons-db/query" "github.com/flanksource/commons-db/query/esdsl" "github.com/flanksource/commons-db/query/providers" ) -func ReadOpenSearch(queryContext dbcontext.Context, r Request, c *models.Connection, ids []string) ([]Span, error) { +// openSearchSource is a connection's span index, read through the span layout +// resolved from its mapping. +type openSearchSource struct { + ctx dbcontext.Context + reference string + options map[string]any + search esdsl.Search + definition layout.Definition + traceIDField string +} + +func resolveOpenSearchSource(queryContext dbcontext.Context, r Request, c *models.Connection) (openSearchSource, error) { if c == nil { - return nil, fmt.Errorf("source connection %q is required", r.Source) + return openSearchSource{}, fmt.Errorf("source connection %q is required", r.Source) } if c.Type != models.ConnectionTypeOpenSearch && c.Type != models.ConnectionTypeOpenTelemetry { - return nil, fmt.Errorf("source %q has unsupported type %q", r.Source, c.Type) + return openSearchSource{}, fmt.Errorf("source %q has unsupported type %q", r.Source, c.Type) } - reference := r.Source + // The connection supplies index and format, which the request may override; + // the trace, span and date fields come from the index's span layout. options := map[string]any{} - for _, key := range []string{"index", "format", "dateField", "traceIdField", "spanIdField", "parentIdField", "parentRefType", "serviceField", "operationField"} { - if value := c.Properties[key]; value != "" { - options[key] = value - } - } if r.Index != "" { options["index"] = r.Index } if r.Format != "" { options["format"] = string(r.Format) } - format, _ := options["format"].(string) - if format == "jaeger" { - jaegerDefaults(options) + definition, err := providers.ResolveSpanLayout(queryContext, providers.SpanLayoutRequest{Connection: r.Source, Options: options}) + if err != nil { + return openSearchSource{}, fmt.Errorf("source %q: %w", r.Source, err) + } + traceIDField, err := definition.SourceField("trace_id") + if err != nil { + return openSearchSource{}, fmt.Errorf("source %q: %w", r.Source, err) + } + spanIDField, err := definition.SourceField("span_id") + if err != nil { + return openSearchSource{}, fmt.Errorf("source %q: %w", r.Source, err) + } + clock := definition.Fields["timestamp"] + options["dateField"], options["traceIdField"], options["spanIdField"] = clock.Source, traceIDField, spanIDField + // The service a row reports is the one a pushed service filter matched on. + if service, ok := definition.Fields["service"]; ok { + options["serviceField"] = service.Source + } + return openSearchSource{ + ctx: queryContext, reference: r.Source, options: options, + // A numeric timestamp bounds a read only once its epoch unit is declared. + search: esdsl.Search{TimeFieldFormat: clock.TimeFieldFormat()}, + definition: definition, traceIDField: traceIDField, + }, nil +} + +// read is the spans meeting query, or every span when it is nil, that start +// between from and until; an empty bound leaves that side open. +func (s openSearchSource) read(query *esdsl.Condition, from, until string) ([]Span, error) { + options := maps.Clone(s.options) + search := s.search + search.Query = query + options["search"] = &search + return readOpenSearch(s.ctx, providers.OpenTelemetryRead{Connection: s.reference, Options: options, From: from, To: until}) +} + +// ReadOpenSearch reads the spans of the traces ids, or of r.TraceID, or else +// every span in r's window. +func ReadOpenSearch(queryContext dbcontext.Context, r Request, c *models.Connection, ids []string) ([]Span, error) { + source, err := resolveOpenSearchSource(queryContext, r, c) + if err != nil { + return nil, err } if len(ids) == 0 && r.TraceID != "" { ids = []string{r.TraceID} } - if len(ids) > 0 { - field := "trace_id" - if value, ok := options["traceIdField"].(string); ok { - field = value + if len(ids) == 0 { + return source.read(nil, r.From, r.Until) + } + byID := termsCondition(source.traceIDField, ids) + return source.read(&byID, r.From, r.Until) +} + +// SearchOpenSearch searches the traces with a span in r's window. Every span +// the window holds settles which traces those are; the filters the span layout +// matches exactly then narrow them across whole traces, so a filter met +// outside the window still counts, and only the traces that may pass are read +// in full. The services facet still covers every trace in the window. +func SearchOpenSearch(queryContext dbcontext.Context, r Request, c *models.Connection) (SearchResult, error) { + source, err := resolveOpenSearchSource(queryContext, r, c) + if err != nil { + return SearchResult{}, err + } + window, err := source.read(nil, r.From, r.Until) + if err != nil { + return SearchResult{}, err + } + window = slices.DeleteFunc(window, func(span Span) bool { return !inWindow(span, r) }) + ids := traceIDs(window) + candidates, err := source.narrow(pushdown(r, source.definition), ids) + if err != nil { + return SearchResult{}, err + } + spans := []Span{} + if len(candidates) > 0 { + byID := termsCondition(source.traceIDField, candidates) + if spans, err = source.read(&byID, "", ""); err != nil { + return SearchResult{}, err } - values := make([]esdsl.Value, len(ids)) - for i, id := range ids { - values[i] = *esdsl.Literal(id) + } + services := spanServices(slices.Concat(window, spans)) + kept := map[string]bool{} + for _, id := range candidates { + kept[id] = true + } + if others := slices.DeleteFunc(ids, func(id string) bool { return kept[id] }); len(others) > 0 { + unseen := source.unseenServices(others, services) + rest, err := source.read(&unseen, "", "") + if err != nil { + return SearchResult{}, err } - options["search"] = &esdsl.Search{Query: &esdsl.Condition{Op: "terms", Field: field, Values: values}} + maps.Copy(services, spanServices(rest)) } - return readOpenSearch(queryContext, providers.OpenTelemetryRead{Connection: reference, Options: options, From: r.From, To: r.Until}) + result, err := Search(spans, r) + if err != nil { + return result, err + } + result.Services = slices.AppendSeq([]string{}, maps.Keys(services)) + slices.Sort(result.Services) + return result, nil } -func jaegerDefaults(options map[string]any) { - for key, value := range map[string]string{"dateField": "startTimeMillis", "traceIdField": "traceID", "spanIdField": "spanID", "parentRefType": "CHILD_OF", "serviceField": "process.serviceName", "operationField": "operationName"} { - if _, ok := options[key]; !ok { - options[key] = value - } +// unseenServices is the spans of the traces ids whose service is none of +// known: all the services of those traces add. Without a keyword service field +// it is every span of them. +func (s openSearchSource) unseenServices(ids []string, known map[string]bool) esdsl.Condition { + condition := esdsl.Condition{Op: esdsl.OpBool, Conditions: []esdsl.Condition{termsCondition(s.traceIDField, ids)}} + if field, ok := keywordField(s.definition, "service"); ok { + seen := termsCondition(field, slices.Sorted(maps.Keys(known))) + seen.Occur = esdsl.OccurMustNot + condition.Conditions = append(condition.Conditions, seen) + } + return condition +} + +func spanServices(spans []Span) map[string]bool { + services := map[string]bool{} + for _, span := range spans { + services[span.Service] = true } + return services } func readOpenSearch(ctx dbcontext.Context, request providers.OpenTelemetryRead) ([]Span, error) { diff --git a/query/tracequery/opensearch_test.go b/query/tracequery/opensearch_test.go index e64b9a81..8beffbc9 100644 --- a/query/tracequery/opensearch_test.go +++ b/query/tracequery/opensearch_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "net/http" "net/http/httptest" - "strings" "time" "github.com/flanksource/commons-db/connection/crd" @@ -26,17 +25,11 @@ var _ = Describe("OpenSearch trace search", func() { return } w.Header().Set("Content-Type", "application/json") - fieldType := "date" + mapping := map[string]string{"@timestamp": "date", "trace_id": "keyword", "span_id": "keyword", "parent_id": "keyword", "operation_name": "keyword", "service_name": "keyword", "duration_ms": "double"} if format == "jaeger" { - fieldType = "long" + mapping = mappingOf(nestedTags) } - if strings.HasSuffix(r.URL.Path, "/_field_caps") { - Expect(json.NewEncoder(w).Encode(map[string]any{"fields": map[string]any{r.URL.Query().Get("fields"): map[string]any{fieldType: map[string]any{"searchable": true, "aggregatable": true}}}})).To(Succeed()) - return - } - if strings.Contains(r.URL.Path, "/_mapping/field/") { - field := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] - Expect(json.NewEncoder(w).Encode(map[string]any{"traces-example": map[string]any{"mappings": map[string]any{field: map[string]any{"full_name": field, "mapping": map[string]any{field: map[string]any{"type": fieldType}}}}}})).To(Succeed()) + if serveMapping(w, r, mapping) { return } var body map[string]any @@ -66,15 +59,15 @@ var _ = Describe("OpenSearch trace search", func() { search, err := crd.FromModel(models.Connection{Name: "search", Type: models.ConnectionTypeOpenSearch, URL: index.URL}) Expect(err).NotTo(HaveOccurred()) Expect(catalog.Save(store.LayerCWD, search)).To(Succeed()) - source := "search" + request := Request{Source: "search", Index: "traces-*", Format: DocumentFormat(format)} if sourceType == "opentelemetry" { - outer, err := crd.FromModel(models.Connection{Name: "traces", Type: models.ConnectionTypeOpenTelemetry, Properties: types.JSONStringMap{"connection": "connection://search"}}) + // The opentelemetry connection owns its index and format. + outer, err := crd.FromModel(models.Connection{Name: "traces", Type: models.ConnectionTypeOpenTelemetry, Properties: types.JSONStringMap{"connection": "connection://search", "index": "traces-*", "format": format}}) Expect(err).NotTo(HaveOccurred()) Expect(catalog.Save(store.LayerCWD, outer)).To(Succeed()) - source = "traces" + request = Request{Source: "traces"} } service := Service{Connections: catalog} - request := Request{Source: source, Index: "traces-*", Format: DocumentFormat(format)} result, err := service.Search(context.Background(), request) Expect(err).NotTo(HaveOccurred()) Expect(result.Traces).To(HaveExactElements(And(HaveField("TraceID", "trace-search"), HaveField("SpanCount", 2)))) diff --git a/query/tracequery/pushdown.go b/query/tracequery/pushdown.go new file mode 100644 index 00000000..996b71dd --- /dev/null +++ b/query/tracequery/pushdown.go @@ -0,0 +1,129 @@ +package tracequery + +import ( + "maps" + "slices" + "strings" + + "github.com/flanksource/commons-db/otel/span/layout" + "github.com/flanksource/commons-db/query/esdsl" +) + +// pushdown is one condition per included attribute key, then one for the +// included services, each of which a passing trace meets on some span of its +// own. Reading the window's traces that meet them in turn narrows a search +// without dropping a trace Search would keep: different keys may hold on +// different spans, so they are never AND-ed within one read. Each compiles +// through the index's span layout, so a nested tag list and tag fields read +// alike. Exclusions, and anything the layout cannot match exactly, stay with +// Search. +const derivedServiceAttribute = "service.name" + +func pushdown(r Request, definition layout.Definition) []esdsl.Condition { + result := []esdsl.Condition{} + groups := r.Attributes.includedByKey() + for _, key := range slices.Sorted(maps.Keys(groups)) { + // Normalization sets resource service.name from the span's service field, + // which no attribute source holds, so only Search can match it. + if key == derivedServiceAttribute { + continue + } + if condition, ok := attributeCondition(definition, groups[key]); ok { + result = append(result, condition) + } + } + if services := r.Services.values(Include); len(services) > 0 { + if field, ok := keywordField(definition, "service"); ok { + slices.Sort(services) + result = append(result, termsCondition(field, services)) + } + } + return result +} + +// attributeCondition is any of exprs on a span or resource attribute, or false +// when one of them has no exact condition in the layout: that key is then +// matched by Search alone, which reads every span the time window holds. +func attributeCondition(definition layout.Definition, exprs []string) (esdsl.Condition, bool) { + should := []esdsl.Condition{} + for _, expr := range slices.Sorted(slices.Values(exprs)) { + op, key, value, ok := attributeOperator(expr) + if !ok { + return esdsl.Condition{}, false + } + values := []string{value} + if op == esdsl.OpExists { + values = nil + } + condition, err := definition.AttributeCondition(key, op, values...) + if err != nil { + return esdsl.Condition{}, false + } + should = append(should, condition.Conditions...) + } + return esdsl.Condition{Op: esdsl.OpBool, Conditions: should}, true +} + +// attributeOperator reads expr as attributeMatches does: a bare key is +// presence, key=value an exact value and key=prefix* a prefix. Any other +// wildcard is not pushed down. +func attributeOperator(expr string) (op esdsl.Operator, key, value string, ok bool) { + key, want, hasValue := strings.Cut(expr, "=") + key, want = strings.TrimSpace(key), strings.TrimSpace(want) + star := strings.Index(want, "*") + switch { + case !hasValue: + return esdsl.OpExists, key, "", true + case star < 0: + return esdsl.OpTerm, key, want, true + case star > 0 && star == len(want)-1: + return esdsl.OpPrefix, key, strings.TrimSuffix(want, "*"), true + default: + return "", "", "", false + } +} + +// keywordField is the field an exact terms condition on canonical reads, when +// the layout maps it to a keyword. +func keywordField(definition layout.Definition, canonical string) (string, bool) { + field, ok := definition.Fields[canonical] + if !ok || field.Type != "keyword" && field.Query == "" { + return "", false + } + name, err := definition.SourceField(canonical) + return name, err == nil +} + +func termsCondition(field string, values []string) esdsl.Condition { + literals := make([]esdsl.Value, len(values)) + for i, value := range values { + literals[i] = *esdsl.Literal(value) + } + return esdsl.Condition{Op: esdsl.OpTerms, Field: field, Values: literals} +} + +// narrow is the traces of ids with a span meeting each condition. Each is read +// across whole traces, never the window, since the span meeting a filter may +// start outside it, and asks only of the traces the read before kept. +func (s openSearchSource) narrow(conditions []esdsl.Condition, ids []string) ([]string, error) { + for _, condition := range conditions { + if len(ids) == 0 { + break + } + query := esdsl.Condition{Op: esdsl.OpBool, Conditions: []esdsl.Condition{condition, termsCondition(s.traceIDField, ids)}} + spans, err := s.read(&query, "", "") + if err != nil { + return nil, err + } + ids = traceIDs(spans) + } + return ids, nil +} + +func traceIDs(spans []Span) []string { + ids := map[string]bool{} + for _, span := range spans { + ids[span.TraceID] = true + } + return slices.Sorted(maps.Keys(ids)) +} diff --git a/query/tracequery/pushdown_test.go b/query/tracequery/pushdown_test.go new file mode 100644 index 00000000..7b52de95 --- /dev/null +++ b/query/tracequery/pushdown_test.go @@ -0,0 +1,393 @@ +package tracequery + +import ( + "cmp" + "context" + "encoding/json" + "maps" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "time" + + "github.com/flanksource/commons-db/connection/crd" + "github.com/flanksource/commons-db/connection/store" + "github.com/flanksource/commons-db/models" + "github.com/flanksource/commons-db/types" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// jaegerCore is the mapping every Jaeger span index shares; the tag shapes +// below add where span and process tags are kept. +var jaegerCore = map[string]string{"traceID": "keyword", "spanID": "keyword", "startTimeMillis": "long", "duration": "long", "operationName": "keyword", "process": "object", "process.serviceName": "keyword"} + +// Jaeger keeps tags as nested key/value lists, or with tags_as_fields as +// tag. keyword fields with "." written as "@". +var ( + nestedTags = map[string]string{"tags": "nested", "tags.key": "keyword", "tags.value": "keyword", "process.tags": "nested", "process.tags.key": "keyword", "process.tags.value": "keyword"} + tagsAsFields = map[string]string{"tag": "object", "process.tag": "object", "tag.oipa@plan@guid": "keyword"} + bothTagShapes = mappingOf(nestedTags, tagsAsFields) +) + +func mappingOf(shapes ...map[string]string) map[string]string { + mapping := maps.Clone(jaegerCore) + for _, shape := range shapes { + maps.Copy(mapping, shape) + } + return mapping +} + +// serveMapping answers the inspection requests a trace read makes from +// mapping: the _field_caps of every field or the named ones, and a named +// field's mapping. It reports whether r was one of them. +func serveMapping(w http.ResponseWriter, r *http.Request, mapping map[string]string) bool { + if strings.HasSuffix(r.URL.Path, "/_field_caps") { + names := strings.Split(r.URL.Query().Get("fields"), ",") + if names[0] == "*" { + names = slices.Collect(maps.Keys(mapping)) + } + fields := map[string]any{} + for _, name := range names { + if typ, ok := mapping[name]; ok { + fields[name] = map[string]any{typ: map[string]any{"searchable": true, "aggregatable": true}} + } + } + Expect(json.NewEncoder(w).Encode(map[string]any{"fields": fields})).To(Succeed()) + return true + } + if strings.Contains(r.URL.Path, "/_mapping/field/") { + field := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + Expect(json.NewEncoder(w).Encode(map[string]any{"traces-example": map[string]any{"mappings": map[string]any{field: map[string]any{"full_name": field, "mapping": map[string]any{field: map[string]any{"type": mapping[field]}}}}}})).To(Succeed()) + return true + } + return false +} + +// jaegerIndex is a fake OpenSearch over Jaeger span documents mapped as +// mapping. It evaluates the bool, nested, term, terms, prefix, exists and +// numeric range clauses a trace read sends, and records each search body's +// query. +type jaegerIndex struct { + mapping map[string]string + docs []map[string]any + mu sync.Mutex + queries []any +} + +func (x *jaegerIndex) ServeHTTP(w http.ResponseWriter, r *http.Request) { + defer GinkgoRecover() + if r.Method == http.MethodHead { + return + } + w.Header().Set("Content-Type", "application/json") + if serveMapping(w, r, x.mapping) { + return + } + var body map[string]any + Expect(json.NewDecoder(r.Body).Decode(&body)).To(Succeed()) + hits := []any{} + if body["search_after"] == nil { + x.mu.Lock() + x.queries = append(x.queries, body["query"]) + x.mu.Unlock() + for _, doc := range x.docs { + if matches(body["query"].(map[string]any), doc) { + hits = append(hits, map[string]any{"_index": "traces-example", "_id": doc["spanID"], "_source": doc, "sort": []any{cmp.Or(doc["startTimeMillis"], doc["startTimeUnixNano"]), doc["spanID"]}}) + } + } + } + Expect(json.NewEncoder(w).Encode(map[string]any{"took": 1, "hits": map[string]any{"total": map[string]any{"value": len(hits), "relation": "eq"}, "hits": hits}})).To(Succeed()) +} + +func matches(clause map[string]any, doc map[string]any) bool { + for op, raw := range clause { + body := raw.(map[string]any) + switch op { + case "bool": + return matchesBool(body, doc) + case "nested": + path := body["path"].(string) + entries, _ := lookup(doc, path).([]any) + return slices.ContainsFunc(entries, func(entry any) bool { + return matches(body["query"].(map[string]any), nest(path, entry)) + }) + case "exists": + return lookup(doc, body["field"].(string)) != nil + case "range": + return matchesRange(body, doc) + case "match_all": + return true + } + for field, want := range body { + got := lookup(doc, field) + switch op { + case "term": + return got == want + case "terms": + return slices.Contains(want.([]any), got) + case "prefix": + text, ok := got.(string) + return ok && strings.HasPrefix(text, want.(string)) + } + } + } + Fail("fake index cannot evaluate " + toJSON(clause)) + return false +} + +func matchesBool(body map[string]any, doc map[string]any) bool { + all := func(key string) bool { + clauses, _ := body[key].([]any) + return !slices.ContainsFunc(clauses, func(c any) bool { return !matches(c.(map[string]any), doc) }) + } + should, _ := body["should"].([]any) + mustNot, _ := body["must_not"].([]any) + return all("filter") && all("must") && !slices.ContainsFunc(mustNot, func(c any) bool { return matches(c.(map[string]any), doc) }) && + (len(should) == 0 || slices.ContainsFunc(should, func(c any) bool { return matches(c.(map[string]any), doc) })) +} + +// matchesRange bounds a numeric time field by the epoch values a read encodes. +func matchesRange(body map[string]any, doc map[string]any) bool { + for field, raw := range body { + got, ok := lookup(doc, field).(float64) + if !ok { + return false + } + for bound, value := range raw.(map[string]any) { + want, numeric := value.(float64) + if !numeric { + continue + } + if !map[string]bool{"gte": got >= want, "gt": got > want, "lte": got <= want, "lt": got < want}[bound] { + return false + } + } + } + return true +} + +func lookup(doc map[string]any, field string) any { + var current any = doc + for _, part := range strings.Split(field, ".") { + object, ok := current.(map[string]any) + if !ok { + return nil + } + current = object[part] + } + return current +} + +func nest(path string, value any) map[string]any { + parts := strings.Split(path, ".") + for i := len(parts) - 1; i > 0; i-- { + value = map[string]any{parts[i]: value} + } + return map[string]any{parts[0]: value} +} + +func toJSON(value any) string { + encoded, err := json.Marshal(value) + Expect(err).NotTo(HaveOccurred()) + return string(encoded) +} + +func jaegerSpan(trace, id, parent, service string, at time.Time, tags ...string) map[string]any { + entries := []any{} + for i := 0; i < len(tags); i += 2 { + entries = append(entries, map[string]any{"key": tags[i], "type": "string", "value": tags[i+1]}) + } + doc := map[string]any{"startTimeMillis": float64(at.UnixMilli()), "traceID": trace, "spanID": id, "operationName": id, "duration": float64(1000), "tags": entries, "process": map[string]any{"serviceName": service, "tags": []any{}}} + if parent != "" { + doc["references"] = []any{map[string]any{"refType": "CHILD_OF", "spanID": parent}} + } + return doc +} + +// jaegerService searches index through an OpenSearch connection holding +// properties. +func jaegerService(index *jaegerIndex, properties types.JSONStringMap) Service { + server := httptest.NewServer(index) + DeferCleanup(server.Close) + catalog, err := store.New(store.Layer{Name: store.LayerCWD, Dir: GinkgoT().TempDir()}) + Expect(err).NotTo(HaveOccurred()) + search, err := crd.FromModel(models.Connection{Name: "search", Type: models.ConnectionTypeOpenSearch, URL: server.URL, Properties: properties}) + Expect(err).NotTo(HaveOccurred()) + Expect(catalog.Save(store.LayerCWD, search)).To(Succeed()) + return Service{Connections: catalog} +} + +// narrowedQuery is the condition a narrowing read pushed beside the trace IDs +// it asks of, which it checks are ids. +func narrowedQuery(query any, ids ...string) string { + filters := query.(map[string]any)["bool"].(map[string]any)["filter"].([]any) + Expect(filters).To(HaveLen(2), toJSON(query)) + Expect(toJSON(filters[1])).To(MatchJSON(toJSON(map[string]any{"terms": map[string]any{"traceID": ids}}))) + return toJSON(filters[0]) +} + +var _ = Describe("OpenSearch trace pushdown", func() { + const plan = "oipa.plan.guid" + at := time.Now().UTC().Add(-time.Minute).Truncate(time.Millisecond) + window := Request{Source: "search", Index: "traces-*", Format: "jaeger", From: at.Add(-time.Minute).Format(time.RFC3339Nano), Until: at.Add(time.Minute).Format(time.RFC3339Nano)} + nestedTag := func(path, inner string) string { + return `{"nested":{"path":"` + path + `","query":{"bool":{"filter":[{"term":{"` + path + `.key":"` + plan + `"}}` + inner + `]}}}}` + } + either := func(span, resource string) string { + return `{"bool":{"minimum_should_match":1,"should":[` + span + `,` + resource + `]}}` + } + + DescribeTable("pushes an included attribute into the read narrowing the window's traces through the index's tag shape", func(expr string, mapping map[string]string, expected string) { + index := &jaegerIndex{mapping: mapping, docs: []map[string]any{jaegerSpan("trace-a", "a-root", "", "oipa", at, plan, "X")}} + request := window + request.Attributes = Modes{expr: Include} + _, err := jaegerService(index, nil).Search(context.Background(), request) + Expect(err).NotTo(HaveOccurred()) + Expect(len(index.queries)).To(BeNumerically(">=", 2)) + Expect(narrowedQuery(index.queries[1], "trace-a")).To(MatchJSON(expected)) + }, + Entry("key=value as a nested term", plan+"=X", mappingOf(nestedTags), + either(nestedTag("tags", `,{"term":{"tags.value":"X"}}`), nestedTag("process.tags", `,{"term":{"process.tags.value":"X"}}`))), + Entry("key=prefix* as a nested prefix", plan+"=AB*", mappingOf(nestedTags), + either(nestedTag("tags", `,{"prefix":{"tags.value":"AB"}}`), nestedTag("process.tags", `,{"prefix":{"process.tags.value":"AB"}}`))), + Entry("a bare key as a nested key term", plan, mappingOf(nestedTags), + either(nestedTag("tags", ""), nestedTag("process.tags", ""))), + Entry("key=value as a tag field term", plan+"=X", mappingOf(tagsAsFields), + either(`{"term":{"tag.oipa@plan@guid":"X"}}`, `{"term":{"process.tag.oipa@plan@guid":"X"}}`)), + Entry("key=prefix* as a tag field prefix", plan+"=AB*", mappingOf(tagsAsFields), + either(`{"prefix":{"tag.oipa@plan@guid":"AB"}}`, `{"prefix":{"process.tag.oipa@plan@guid":"AB"}}`)), + Entry("a bare key as a tag field exists", plan, mappingOf(tagsAsFields), + either(`{"exists":{"field":"tag.oipa@plan@guid"}}`, `{"exists":{"field":"process.tag.oipa@plan@guid"}}`)), + Entry("key=value as either shape on an index holding both", plan+"=X", bothTagShapes, + `{"bool":{"minimum_should_match":1,"should":[{"term":{"tag.oipa@plan@guid":"X"}},`+nestedTag("tags", `,{"term":{"tags.value":"X"}}`)+ + `,{"term":{"process.tag.oipa@plan@guid":"X"}},`+nestedTag("process.tags", `,{"term":{"process.tags.value":"X"}}`)+`]}}`), + ) + + It("ORs the values of one key and leaves exclusions and inexact wildcards in memory", func() { + index := &jaegerIndex{mapping: mappingOf(nestedTags), docs: []map[string]any{jaegerSpan("trace-a", "a-root", "", "oipa", at, plan, "A")}} + request := window + request.Attributes = Modes{plan + "=A": Include, plan + "=B": Include, "oipa.rule.name": Exclude, "oipa.outcome=*ed": Include} + _, err := jaegerService(index, nil).Search(context.Background(), request) + Expect(err).NotTo(HaveOccurred()) + Expect(index.queries).To(HaveLen(3)) + Expect(narrowedQuery(index.queries[1], "trace-a")).To(MatchJSON(`{"bool":{"minimum_should_match":1,"should":[` + + strings.Join([]string{ + nestedTag("tags", `,{"term":{"tags.value":"A"}}`), nestedTag("process.tags", `,{"term":{"process.tags.value":"A"}}`), + nestedTag("tags", `,{"term":{"tags.value":"B"}}`), nestedTag("process.tags", `,{"term":{"process.tags.value":"B"}}`), + }, ",") + `]}}`)) + }) + + It("leaves the by-trace-ID read unchanged", func() { + index := &jaegerIndex{mapping: mappingOf(nestedTags)} + request := window + request.From, request.Until = "", "" + request.Attributes = Modes{plan + "=X": Include} + _, err := jaegerService(index, nil).ReadSpans(context.Background(), request, []string{"trace-a"}) + Expect(err).NotTo(HaveOccurred()) + Expect(toJSON(index.queries)).To(MatchJSON(`[{"terms":{"traceID":["trace-a"]}}]`)) + }) + + It("finds traces whose spans meet different keys and services by narrowing one read at a time", func() { + index := &jaegerIndex{mapping: mappingOf(nestedTags), docs: []map[string]any{ + jaegerSpan("trace-a", "a-root", "", "oipa", at, plan, "X"), + jaegerSpan("trace-a", "a-rule", "a-root", "rules", at.Add(time.Millisecond), "oipa.rule.name", "Premium"), + jaegerSpan("trace-a", "a-jdbc", "a-root", "db", at.Add(2*time.Millisecond)), + jaegerSpan("trace-b", "b-root", "", "oipa", at, plan, "X"), + jaegerSpan("trace-c", "c-root", "", "oipa", at, plan, "Y"), + jaegerSpan("trace-c", "c-rule", "c-root", "rules", at.Add(time.Millisecond), "oipa.rule.name", "Premium"), + }} + request := window + request.Attributes = Modes{plan + "=X": Include, "oipa.rule.name=Prem*": Include} + request.Services = Modes{"rules": Include} + result, err := jaegerService(index, nil).Search(context.Background(), request) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Traces).To(HaveExactElements(And(HaveField("TraceID", "trace-a"), HaveField("SpanCount", 3)))) + Expect(result.Services).To(Equal([]string{"db", "oipa", "rules"})) + // The window read, one narrowing read per key and the services, the + // full read of the one trace left, then the services the rest add. + Expect(index.queries).To(HaveLen(6)) + Expect(toJSON(index.queries[0])).To(And(ContainSubstring(`"range"`), Not(ContainSubstring(`"terms"`)))) + narrowedQuery(index.queries[1], "trace-a", "trace-b", "trace-c") + narrowedQuery(index.queries[2], "trace-a", "trace-b") + Expect(narrowedQuery(index.queries[3], "trace-a")).To(MatchJSON(`{"terms":{"process.serviceName":["rules"]}}`)) + Expect(toJSON(index.queries[4])).To(MatchJSON(`{"terms":{"traceID":["trace-a"]}}`)) + Expect(toJSON(index.queries[5])).To(MatchJSON(`{"bool":{"filter":[{"terms":{"traceID":["trace-b","trace-c"]}}],"must_not":[{"terms":{"process.serviceName":["db","oipa","rules"]}}]}}`)) + }) + + It("keeps a trace in the window whose filter span starts before it, and facets every trace in the window", func() { + index := &jaegerIndex{mapping: mappingOf(nestedTags), docs: []map[string]any{ + jaegerSpan("trace-x", "x-root", "", "checkout", at.Add(-2*time.Minute)), + jaegerSpan("trace-x", "x-db", "x-root", "db", at), + jaegerSpan("trace-y", "y-root", "", "inventory", at), + jaegerSpan("trace-z", "z-root", "", "payments", at), + jaegerSpan("trace-z", "z-bill", "z-root", "billing", at.Add(2*time.Minute)), + }} + request := window + request.Services = Modes{"checkout": Include} + result, err := jaegerService(index, nil).Search(context.Background(), request) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Traces).To(HaveExactElements(And(HaveField("TraceID", "trace-x"), HaveField("SpanCount", 2)))) + Expect(result.Services).To(Equal([]string{"billing", "checkout", "db", "inventory", "payments"})) + }) + + It("keeps a filter on the service.name normalization derives from the span's service in memory", func() { + index := &jaegerIndex{mapping: mappingOf(nestedTags), docs: []map[string]any{ + jaegerSpan("trace-a", "a-root", "", "checkout", at), + jaegerSpan("trace-b", "b-root", "", "inventory", at), + }} + request := window + request.Attributes = Modes{"service.name=checkout": Include} + result, err := jaegerService(index, nil).Search(context.Background(), request) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Traces).To(HaveExactElements(HaveField("TraceID", "trace-a"))) + // The window read and the full read of both traces: nothing narrowed. + Expect(index.queries).To(HaveLen(2)) + }) + + It("reports the service the connection's span layout names, which a service filter matches", func() { + mapping := mappingOf(nestedTags) + mapping["custom_service"] = "keyword" + span := jaegerSpan("trace-s", "s-root", "", "jaeger-service", at) + span["custom_service"] = "checkout" + index := &jaegerIndex{mapping: mapping, docs: []map[string]any{span}} + request := window + request.Services = Modes{"checkout": Include} + result, err := jaegerService(index, types.JSONStringMap{"span_layout": `{"fields":{"service":{"source":"custom_service"}}}`}).Search(context.Background(), request) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Traces).To(HaveExactElements(And(HaveField("TraceID", "trace-s"), HaveField("Service", "checkout")))) + Expect(result.Services).To(Equal([]string{"checkout"})) + }) + + It("bounds a window on a numeric timestamp in its source unit", func() { + mapping := map[string]string{"traceID": "keyword", "spanID": "keyword", "startTimeUnixNano": "long", "durationNano": "long", "operationName": "keyword", "process": "object", "process.serviceName": "keyword"} + span := func(trace string, started time.Time) map[string]any { + return map[string]any{"traceID": trace, "spanID": trace + "-root", "startTimeUnixNano": float64(started.UnixNano()), "durationNano": float64(time.Millisecond), "operationName": "GET /", "process": map[string]any{"serviceName": "web"}} + } + index := &jaegerIndex{mapping: mapping, docs: []map[string]any{span("trace-in", at), span("trace-before", at.Add(-2*time.Minute))}} + request := window + request.Format = "" + result, err := jaegerService(index, nil).Search(context.Background(), request) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Traces).To(HaveExactElements(HaveField("TraceID", "trace-in"))) + }) + + It("reads spans whose trace ID the connection's span layout names", func() { + mapping := mappingOf(nestedTags) + delete(mapping, "traceID") + mapping["custom_trace_id"] = "keyword" + span := jaegerSpan("trace-k", "k-root", "", "web", at) + span["custom_trace_id"] = span["traceID"] + delete(span, "traceID") + index := &jaegerIndex{mapping: mapping, docs: []map[string]any{span}} + service := jaegerService(index, types.JSONStringMap{"span_layout": `{"fields":{"trace_id":{"source":"custom_trace_id"}}}`}) + result, err := service.Search(context.Background(), window) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Traces).To(HaveExactElements(HaveField("TraceID", "trace-k"))) + detail, err := service.Trace(context.Background(), window, "trace-k") + Expect(err).NotTo(HaveOccurred()) + Expect(detail.Spans).To(HaveExactElements(HaveField("SpanID", "k-root"))) + }) +}) diff --git a/query/tracequery/service.go b/query/tracequery/service.go index 45ecb671..b41ffefa 100644 --- a/query/tracequery/service.go +++ b/query/tracequery/service.go @@ -42,7 +42,14 @@ func (s Service) Search(ctx context.Context, r Request) (SearchResult, error) { if err != nil { return SearchResult{}, err } - spans, err := s.load(ctx, r, nil) + if s.Read == nil { + queryContext, c, err := s.connection(ctx, r.Source) + if err != nil { + return SearchResult{}, err + } + return SearchOpenSearch(queryContext, r, c) + } + spans, err := s.Read(ctx, r, nil) if err != nil { return SearchResult{}, err } @@ -55,7 +62,7 @@ func (s Service) Search(ctx context.Context, r Request) (SearchResult, error) { if len(ids) > 0 { full := r full.From, full.Until = "", "" - spans, err = s.load(ctx, full, ids) + spans, err = s.Read(ctx, full, ids) if err != nil { return SearchResult{}, err } @@ -83,20 +90,38 @@ func (s Service) load(ctx context.Context, r Request, ids []string) ([]Span, err if s.Read != nil { return s.Read(ctx, r, ids) } - if r.Source == "" { - return nil, fmt.Errorf("source is required") + queryContext, c, err := s.connection(ctx, r.Source) + if err != nil { + return nil, err } - if s.Connections == nil { - return nil, fmt.Errorf("trace source catalog is not configured") + return ReadOpenSearch(queryContext, r, c, ids) +} + +func (s Service) connection(ctx context.Context, source string) (dbcontext.Context, *models.Connection, error) { + if source == "" { + return dbcontext.Context{}, nil, fmt.Errorf("source is required") + } + var ( + queryContext dbcontext.Context + c *models.Connection + err error + ) + switch { + case s.Resolve != nil: + queryContext, c, err = s.Resolve(ctx, source) + case s.Connections != nil: + queryContext = s.Connections.Attach(dbcontext.NewContext(ctx)) + c, err = s.Connections.ResolveConnection(dbcontext.NewContext(ctx), source) + default: + return dbcontext.Context{}, nil, fmt.Errorf("trace source catalog is not configured") } - c, err := s.Connections.ResolveConnection(dbcontext.NewContext(ctx), r.Source) if err != nil { - return nil, err + return dbcontext.Context{}, nil, err } if c == nil { - return nil, fmt.Errorf("source connection %q not found", r.Source) + return dbcontext.Context{}, nil, fmt.Errorf("source connection %q not found", source) } - return ReadOpenSearch(s.Connections.Attach(dbcontext.NewContext(ctx)), r, c, ids) + return queryContext, c, nil } // ReadSpans reads the spans selected by a host without assembling a search. diff --git a/query/tracequery/types.go b/query/tracequery/types.go index 785cd552..306d334e 100644 --- a/query/tracequery/types.go +++ b/query/tracequery/types.go @@ -6,6 +6,8 @@ import ( "time" "github.com/flanksource/commons-db/connection/store" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/models" "github.com/flanksource/commons-db/otel/span" ) @@ -87,7 +89,14 @@ type Detail struct { MissingParents []string `json:"missingParents"` } +// Service searches the spans of an OpenSearch or opentelemetry connection, +// or those Read returns. type Service struct { Connections *store.Store - Read func(context.Context, Request, []string) ([]Span, error) + // Resolve is the connection a source names and the context it is read in, + // for a host whose connections Connections does not hold. + Resolve func(context.Context, string) (dbcontext.Context, *models.Connection, error) + // Read replaces the connection read with a host's own spans, which a search + // then filters in memory. + Read func(context.Context, Request, []string) ([]Span, error) } diff --git a/schemas/connection.json b/schemas/connection.json index a09f2749..0104655d 100644 --- a/schemas/connection.json +++ b/schemas/connection.json @@ -6984,6 +6984,22 @@ "x-clicky-order": 2, "x-col-span": "full" }, + "format": { + "description": "How span documents are shaped: flat OpenTelemetry fields, or Jaeger's span documents.", + "enum": [ + "flat", + "jaeger" + ], + "title": "Document format", + "type": "string", + "x-clicky-order": 4, + "x-col-span": "full", + "x-enum-display": "segmented", + "x-enum-labels": { + "flat": "Flat", + "jaeger": "Jaeger" + } + }, "index": { "title": "Default index", "type": "string", @@ -7008,7 +7024,7 @@ "format": "textarea", "title": "Span layout overrides", "type": "string", - "x-clicky-order": 4, + "x-clicky-order": 5, "x-col-span": "full" } }, diff --git a/schemas/src/connections/opentelemetry.json b/schemas/src/connections/opentelemetry.json index ce2d34ca..5752d7d9 100644 --- a/schemas/src/connections/opentelemetry.json +++ b/schemas/src/connections/opentelemetry.json @@ -221,6 +221,22 @@ "x-clicky-order": 2, "x-col-span": "full" }, + "format": { + "description": "How span documents are shaped: flat OpenTelemetry fields, or Jaeger's span documents.", + "enum": [ + "flat", + "jaeger" + ], + "title": "Document format", + "type": "string", + "x-clicky-order": 4, + "x-col-span": "full", + "x-enum-display": "segmented", + "x-enum-labels": { + "flat": "Flat", + "jaeger": "Jaeger" + } + }, "index": { "title": "Default index", "type": "string", @@ -245,7 +261,7 @@ "format": "textarea", "title": "Span layout overrides", "type": "string", - "x-clicky-order": 4, + "x-clicky-order": 5, "x-col-span": "full" } },