Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,11 @@ jobs:

- name: Lint with gavel
uses: flanksource/gavel@4ec24346ade41681a5f1db1de47acd32c32e0879 # v0.0.83
# gavel installs golangci-lint by resolving its stable release through the
# GitHub API; unauthenticated, a shared runner hits the rate limit and the
# install fails, so the read-only workflow token authenticates it.
env:
GITHUB_TOKEN: ${{ github.token }}
with:
args: lint --timeout 10m
artifact-name: gavel-lint
Expand Down
78 changes: 6 additions & 72 deletions cmd/query/connections/browser_traces.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,10 @@ package connections

import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"time"

"github.com/flanksource/clicky/rpc"
dbcontext "github.com/flanksource/commons-db/context"
"github.com/flanksource/commons-db/models"
"github.com/flanksource/commons-db/query/tracequery"
)
Expand All @@ -21,70 +15,10 @@ func (h *connectionBrowserHandler) serveTraces(w http.ResponseWriter, r *http.Re
http.Error(w, "connection does not support OpenSearch traces", 400)
return
}
if r.Header.Get("Accept") == "application/schema+json" {
writeJSON(w, rpc.SchemaForStruct(tracequery.Request{}))
return
}
request, err := browserTraceRequest(r)
if err != nil {
http.Error(w, err.Error(), 400)
return
}
request.Source = conn.ID.String()
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
service := tracequery.Service{Read: func(ctx context.Context, request tracequery.Request, ids []string) ([]tracequery.Span, error) {
queryContext := withBrowserConnection(h.ctx.Wrap(ctx), conn)
return tracequery.ReadOpenSearch(queryContext, request, conn, ids)
service := tracequery.Service{Resolve: func(ctx context.Context, _ string) (dbcontext.Context, *models.Connection, error) {
return withBrowserConnection(h.ctx.Wrap(ctx), conn), conn, nil
}}
var result any
if tail == "/traces" {
result, err = service.Search(ctx, request)
} else {
result, err = service.Trace(ctx, request, strings.TrimPrefix(tail, "/traces/"))
}
if err != nil {
status := http.StatusBadRequest
if errors.Is(err, tracequery.ErrTraceNotFound) {
status = http.StatusNotFound
}
http.Error(w, err.Error(), status)
return
}
writeJSON(w, result)
}

func browserTraceRequest(r *http.Request) (tracequery.Request, error) {
schema := rpc.SchemaForStruct(tracequery.Request{})
body := map[string]any{}
values, err := url.ParseQuery(r.URL.RawQuery)
if err != nil {
return tracequery.Request{}, err
}
for key, items := range values {
property, ok := schema.Properties[key]
if !ok || len(items) != 1 {
return tracequery.Request{}, fmt.Errorf("unknown or repeated trace parameter %q", key)
}
switch property.Type {
case "integer":
body[key], err = strconv.Atoi(items[0])
case "object":
var modes map[string]string
err = json.Unmarshal([]byte(items[0]), &modes)
body[key] = modes
default:
body[key] = items[0]
}
if err != nil {
return tracequery.Request{}, fmt.Errorf("%s: %w", key, err)
}
}
data, err := json.Marshal(body)
if err != nil {
return tracequery.Request{}, err
}
var request tracequery.Request
err = json.Unmarshal(data, &request)
return request, err
traces := r.Clone(r.Context())
traces.URL.Path, traces.URL.RawPath = strings.TrimPrefix(tail, "/traces"), ""
tracequery.Handler(service, conn.ID.String()).ServeHTTP(w, traces)
}
11 changes: 10 additions & 1 deletion cmd/query/connections/browser_traces_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,16 @@ var _ = Describe("OpenSearch trace browser integration", func() {
return
}
if strings.HasSuffix(r.URL.Path, "/_field_caps") {
Expect(json.NewEncoder(w).Encode(map[string]any{"fields": map[string]any{"@timestamp": map[string]any{"date": map[string]any{"searchable": true, "aggregatable": true}}}})).To(Succeed())
// The span layout is discovered from every mapped field; the time
// range only inspects the date field.
mapping := map[string]string{"@timestamp": "date", "trace_id": "keyword", "span_id": "keyword", "parent_id": "keyword", "service_name": "keyword", "operation_name": "keyword", "duration_ms": "double"}
fields := map[string]any{}
for name, typ := range mapping {
if requested := r.URL.Query().Get("fields"); requested == "*" || requested == name {
fields[name] = map[string]any{typ: map[string]any{"searchable": true, "aggregatable": true}}
}
}
Expect(json.NewEncoder(w).Encode(map[string]any{"fields": fields})).To(Succeed())
return
}
if strings.Contains(r.URL.Path, "/_mapping/field/") {
Expand Down
13 changes: 0 additions & 13 deletions cmd/query/connections/browser_traces_test.go
Original file line number Diff line number Diff line change
@@ -1,25 +1,12 @@
package connections

import (
"net/http"
"net/http/httptest"

"github.com/flanksource/clicky/rpc"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var _ = Describe("OpenSearch trace HTTP contract", func() {
It("decodes typed trace filters and rejects malformed parameters", func() {
request, err := browserTraceRequest(httptest.NewRequest(http.MethodGet, "/traces?minSpans=2&services=%7B%22checkout%22:%22include%22%7D", nil))
Expect(err).NotTo(HaveOccurred())
Expect(request.MinSpans).To(Equal(2))
Expect(string(request.Services["checkout"])).To(Equal("include"))
for _, query := range []string{"minSpans=oops", "unknown=value", "limit=1&limit=2", "services=invalid"} {
_, err := browserTraceRequest(httptest.NewRequest(http.MethodGet, "/traces?"+query, nil))
Expect(err).To(HaveOccurred())
}
})
It("documents search and detail with their typed response schemas", func() {
spec := &rpc.OpenAPISpec{}
AddConnectionsOpenAPI(spec)
Expand Down
2 changes: 1 addition & 1 deletion cmd/query/connections/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ func (s *Service) withVirtual(stored []*models.Connection, options ListOptions)
stored = slices.DeleteFunc(stored, func(existing *models.Connection) bool {
return existing.Namespace == connection.Namespace && existing.Name == connection.Name
})
if types := connectionTypeFilter(options); len(types) > 0 && !slices.Contains(types, connection.Type) && !(connection.Type == "" && len(connection.Warnings) > 0) {
if types := connectionTypeFilter(options); len(types) > 0 && !slices.Contains(types, connection.Type) && (connection.Type != "" || len(connection.Warnings) == 0) {
continue
}
connection = cloneConnection(connection)
Expand Down
2 changes: 1 addition & 1 deletion cmd/query/connections/store_layers.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,5 +97,5 @@ func (p *StoreProvider) saveToLayer(ctx context.Context, c *models.Connection) (
if saved == nil {
return nil, fmt.Errorf("saved connection %q cannot be resolved", c.Name)
}
return p.project(*saved, true), nil
return p.project(*saved), nil
}
12 changes: 5 additions & 7 deletions cmd/query/connections/store_provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,9 @@ func (p *StoreProvider) ListConnections() ([]*models.Connection, error) {
if err != nil {
return nil, err
}
writable := p.Writable()
listed := make([]*models.Connection, 0, len(entries))
for _, entry := range entries {
listed = append(listed, p.project(entry, writable))
listed = append(listed, p.project(entry))
}
return listed, nil
}
Expand All @@ -90,18 +89,17 @@ func (p *StoreProvider) ResolveConnection(_ dbcontext.Context, reference string)
if err != nil || entry == nil {
return nil, err
}
return p.project(*entry, p.Writable()), nil
return p.project(*entry), nil
}

// project is entry's connection as the service sees it: a write-layer
// connection is editable while the layer is writable; another layer's stays
// virtual and read-only, so the service refuses to edit it.
func (p *StoreProvider) project(entry store.Entry, writable bool) *models.Connection {
func (p *StoreProvider) project(entry store.Entry) *models.Connection {
connection := entry.Connection
connection.Properties = maps.Clone(connection.Properties)
if entry.Layer.Name == p.layer.Name || slices.Contains(p.writableLayers, entry.Layer.Name) {
writable = dirWritable(entry.Layer.Dir)
connection.Virtual, connection.ReadOnly = false, !writable
connection.Virtual, connection.ReadOnly = false, !dirWritable(entry.Layer.Dir)
} else {
connection.Virtual, connection.ReadOnly = true, true
}
Expand Down Expand Up @@ -165,7 +163,7 @@ func (p *StoreProvider) SaveConnection(ctx context.Context, connection *models.C
if saved == nil {
return nil, fmt.Errorf("connection %q was saved but the store does not resolve it", resource.Name)
}
return p.project(*saved, true), nil
return p.project(*saved), nil
}

// DeleteConnection removes the write layer's file for the connection id names.
Expand Down
2 changes: 1 addition & 1 deletion cmd/query/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ require (
github.com/spf13/pflag v1.0.10
github.com/stretchr/testify v1.12.1
github.com/valkey-io/valkey-go v1.0.79
go.yaml.in/yaml/v3 v3.0.5
gocloud.dev v0.46.0
golang.org/x/sync v0.24.0
gorm.io/gorm v1.31.2
Expand Down Expand Up @@ -361,6 +360,7 @@ require (
go.opentelemetry.io/otel/trace v1.47.0 // indirect
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.4 // indirect
golang.org/x/crypto v0.58.0 // indirect
golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3 // indirect
Expand Down
2 changes: 1 addition & 1 deletion cmd/query/localfiles/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func Handler(prefix string, next http.Handler) http.Handler {
}
if r.Method != http.MethodGet {
w.Header().Set("Allow", "GET")
http.Error(w, "method not allowed", 405)
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
listing, err := list(r.URL.Query().Get("path"))
Expand Down
6 changes: 5 additions & 1 deletion cmd/query/profiles/family.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,11 @@ func (s *Service) listSurfaces(ctx context.Context) ([]entity.DynamicEntitySpec,
// Every instance listed here is documented with a run, and the profile
// extension has already described one that has none — so listing it would
// both advertise a run that can only fail and duplicate its surface.
if !advertisesRun(resolved.Profile) {
runs, err := advertisesRun(resolved.Profile)
if err != nil {
return nil, fmt.Errorf("profile surface %q: %w", profile.Name, err)
}
if !runs {
continue
}
surface, err := s.profileSurface(resolved.Profile)
Expand Down
16 changes: 6 additions & 10 deletions cmd/query/profiles/family_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -271,17 +271,13 @@ func TestProfileFamilyResolvesAProfileCreatedAfterTheRoutesWereBuilt(t *testing.

// The served document is the family's paths laid over the profile extension's,
// and the family fills any profile path the extension left empty. A trace or a
// legacy placeholder is left without a run on purpose — running it can only
// catalog-only profile is left without a run on purpose — running it can only
// fail — so the family must not put one back, nor list its surface twice.
func TestProfileFamilyAdvertisesARunOnlyForAProfileThatRuns(t *testing.T) {
legacy, err := ConvertLegacyProfile([]byte("name: legacy-sql\nkind: sql_xevent\n"))
if err != nil {
t.Fatal(err)
}
capture := lookupProfile("Capture")
capture.Trace = &query.TraceSpec{}
query.RegisterProvider(familyLookupMock{})
service, _ := newProfileServiceTest(t, lookupProfile("Spans"), capture, legacy)
service, _ := newProfileServiceTest(t, lookupProfile("Spans"), capture, catalogOnlyProfile("catalog-sql"))
service.RegisterFamily()
t.Cleanup(func() { entity.UnregisterDynamicEntityFamily(profileFamilyName) })

Expand Down Expand Up @@ -313,7 +309,7 @@ func TestProfileFamilyAdvertisesARunOnlyForAProfileThatRuns(t *testing.T) {
Surfaces int
}
got := map[string]advertised{}
for _, key := range []string{"profile-spans", "profile-capture", "profile-legacy-sql"} {
for _, key := range []string{"profile-spans", "profile-capture", "profile-catalog-sql"} {
_, run := spec.Paths["/api/v1/profile/"+key]["get"]
_, start := spec.Paths["/api/v1/profile/"+key+"/sessions"]["post"]
surfaces := 0
Expand All @@ -325,9 +321,9 @@ func TestProfileFamilyAdvertisesARunOnlyForAProfileThatRuns(t *testing.T) {
got[key] = advertised{Run: run, SessionStart: start, Surfaces: surfaces}
}
want := map[string]advertised{
"profile-spans": {Run: true, SessionStart: false, Surfaces: 1},
"profile-capture": {Run: false, SessionStart: true, Surfaces: 1},
"profile-legacy-sql": {Run: false, SessionStart: false, Surfaces: 1},
"profile-spans": {Run: true, SessionStart: false, Surfaces: 1},
"profile-capture": {Run: false, SessionStart: true, Surfaces: 1},
"profile-catalog-sql": {Run: false, SessionStart: false, Surfaces: 1},
}
for key, expected := range want {
if got[key] != expected {
Expand Down
Loading
Loading