From e06d38fd5c71ae47274ebd002efdb1f059ed9bf7 Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Tue, 10 Nov 2026 20:11:12 +0530 Subject: [PATCH 1/8] feat(substrate): list the actors of one atespace or every atespace, and the workers ListActors and ListWorkers report what a substrate holds, so a dashboard can show worker capacity against allocation and actors by state. Sweep lists through ListActors. The SPIFFE ID helper is SPIFFEID, freeing Actor for the listed type. --- pkg/substrate/README.md | 5 ++ pkg/substrate/e2e_test.go | 19 ++++- pkg/substrate/inventory.go | 117 ++++++++++++++++++++++++++ pkg/substrate/inventory_test.go | 67 +++++++++++++++ pkg/substrate/substrate.go | 28 +++--- pkg/substrate/substratetest/server.go | 88 ++++++++++++++----- 6 files changed, 284 insertions(+), 40 deletions(-) create mode 100644 pkg/substrate/inventory.go create mode 100644 pkg/substrate/inventory_test.go diff --git a/pkg/substrate/README.md b/pkg/substrate/README.md index eee40c65..40f6a0fe 100644 --- a/pkg/substrate/README.md +++ b/pkg/substrate/README.md @@ -27,6 +27,11 @@ An `Env` moves files in and out (`WriteFile`, `ReadFile`, both streamed) and sta stdin of a process started with `Stdin` (`Input`, one call per write, so an ended stream loses nothing). It waits for its exit code (`Wait`) and kills its process group (`Kill`). `Delete` deletes the environment. +`Client.ListActors(atespace)` lists the actors of an atespace, or of every atespace when it is +empty, with their state and create time. `Client.ListWorkers()` lists the workers they run on, with +each one's pool, node, state, and capacity and allocation (CPU, memory and actor slots). `Sweep` +deletes the environments a dead caller left behind, from the atespace's actor list. + ## Generated API `internal/ateapi`, `internal/credprovider` and `internal/ateenvv1alpha` are generated by `go diff --git a/pkg/substrate/e2e_test.go b/pkg/substrate/e2e_test.go index acd3b772..944c8a19 100644 --- a/pkg/substrate/e2e_test.go +++ b/pkg/substrate/e2e_test.go @@ -1,5 +1,5 @@ // The substrate client against a live cluster: files go in and come out of an environment intact, -// whatever their size, a process in it works on them, and the caller may list what a sweep needs. +// whatever their size, a process in it works on them, and the caller may list what a sweep and an inventory need. package substrate_test import ( @@ -98,3 +98,20 @@ func TestSweepListsTheActorsOfACluster(t *testing.T) { require.NoError(t, err) assert.Empty(t, swept) } + +// A caller's identity may list the cluster's workers and the actors of every atespace, as an +// inventory of the substrate must. +func TestInventoryOfACluster(t *testing.T) { + cluster := substratetest.FromEnv(t) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + client, err := substrate.Dial(cluster.Options) + require.NoError(t, err) + defer client.Close() + + workers, err := client.ListWorkers(ctx) + require.NoError(t, err) + assert.NotEmpty(t, workers, "a cluster that runs environments has workers") + _, err = client.ListActors(ctx, "") + require.NoError(t, err) +} diff --git a/pkg/substrate/inventory.go b/pkg/substrate/inventory.go new file mode 100644 index 00000000..90962c8d --- /dev/null +++ b/pkg/substrate/inventory.go @@ -0,0 +1,117 @@ +// Lists what a substrate holds: the actors of one atespace or of every atespace, and the workers +// they run on, with each worker's capacity and how much of it its actors are allocated. +package substrate + +import ( + "context" + "fmt" + "strings" + "time" + + ateapipb "github.com/flanksource/captain/pkg/substrate/internal/ateapi" +) + +// Actor is one actor in a substrate: an environment, or anything else Substrate runs. +type Actor struct { + Atespace, Name string + // State is resuming, running, suspending, suspended, crashed, deleting or reverting. + State string + // Created is when the actor was created; zero when Substrate doesn't say. + Created time.Time +} + +// Worker is one worker pod that actors run on. +type Worker struct { + Name, Pool, Node string + // State is active, draining or unavailable. + State string + // Capacity is what the worker can hold, and Allocated what its actors are allocated of it, + // their runtime's overhead included. + Capacity, Allocated Resources +} + +// Resources is an amount of a worker's CPU, memory and actor slots. +type Resources struct { + CPUMillicores int64 + MemoryBytes int64 + Actors int32 +} + +// listPageSize is how many items one list call asks Substrate for. +const listPageSize = 100 + +// ListActors lists the actors of atespace, or of every atespace when atespace is empty. +func (c *Client) ListActors(ctx context.Context, atespace string) ([]Actor, error) { + request := &ateapipb.ListActorsRequest{Atespace: atespace, Options: &ateapipb.ListOptions{PageSize: listPageSize}} + return list(func(token string) ([]Actor, string, error) { + request.Options.PageToken = token + page, err := c.control.ListActors(ctx, request) + if err != nil { + if atespace == "" { + return nil, "", fmt.Errorf("list the actors of every atespace: %w", err) + } + return nil, "", fmt.Errorf("list the actors of atespace %s: %w", atespace, err) + } + actors := make([]Actor, 0, len(page.GetActors())) + for _, actor := range page.GetActors() { + metadata := actor.GetMetadata() + listed := Actor{Atespace: metadata.GetAtespace(), Name: metadata.GetName(), State: stateName(actor.GetStatus().GetState().String(), "ACTOR_STATE_")} + if metadata.GetCreateTime() != nil { + listed.Created = metadata.GetCreateTime().AsTime() + } + actors = append(actors, listed) + } + return actors, page.GetNextPageToken(), nil + }) +} + +// ListWorkers lists the substrate's workers. +func (c *Client) ListWorkers(ctx context.Context) ([]Worker, error) { + request := &ateapipb.ListWorkersRequest{Options: &ateapipb.ListOptions{PageSize: listPageSize}} + return list(func(token string) ([]Worker, string, error) { + request.Options.PageToken = token + page, err := c.control.ListWorkers(ctx, request) + if err != nil { + return nil, "", fmt.Errorf("list substrate's workers: %w", err) + } + workers := make([]Worker, 0, len(page.GetWorkers())) + for _, worker := range page.GetWorkers() { + status := worker.GetStatus() + workers = append(workers, Worker{ + Name: worker.GetMetadata().GetName(), Pool: worker.GetWorkerPool(), Node: worker.GetNodeName(), + State: stateName(status.GetState().String(), "WORKER_STATE_"), + Capacity: resources(status.GetCapacity()), + Allocated: resources(status.GetAllocated()), + }) + } + return workers, page.GetNextPageToken(), nil + }) +} + +// list collects every page that page returns, starting from the empty token and following each +// page's next token until it is empty. +func list[T any](page func(token string) ([]T, string, error)) ([]T, error) { + var all []T + token := "" + for { + items, next, err := page(token) + if err != nil { + return nil, err + } + all = append(all, items...) + if next == "" { + return all, nil + } + token = next + } +} + +// stateName is the lower-case name of a Substrate enum value without its prefix, e.g. running +// for ACTOR_STATE_RUNNING. +func stateName(value, prefix string) string { + return strings.ToLower(strings.TrimPrefix(value, prefix)) +} + +func resources(r *ateapipb.WorkerResources) Resources { + return Resources{CPUMillicores: r.GetCpuMillicores(), MemoryBytes: r.GetMemoryBytes(), Actors: r.GetActors()} +} diff --git a/pkg/substrate/inventory_test.go b/pkg/substrate/inventory_test.go new file mode 100644 index 00000000..fe4db117 --- /dev/null +++ b/pkg/substrate/inventory_test.go @@ -0,0 +1,67 @@ +// Specs for what a substrate holds, against an in-process Substrate: the actors of one atespace or +// of all of them, and the workers with their capacity and what actors use of it. +package substrate_test + +import ( + "testing" + "time" + + "github.com/flanksource/captain/pkg/substrate" + "github.com/flanksource/captain/pkg/substrate/substratetest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestListActorsListsOneAtespaceOrEveryAtespace(t *testing.T) { + server := substratetest.Start(t) + server.ListPageSize = 1 + ctx := testContext(t) + for atespace, ids := range map[string][]string{"ate-env": {"run-2", "run-1"}, "gavel": {"run-3"}} { + client := dial(t, server, func(o *substrate.Options) { o.Atespace = atespace }) + for _, id := range ids { + _, err := client.Create(ctx, substrate.Environment{ID: id}) + require.NoError(t, err) + } + } + client := dial(t, server) + names := func(actors []substrate.Actor) []string { + var names []string + for _, actor := range actors { + names = append(names, actor.Atespace+"/"+actor.Name) + } + return names + } + + inOne, err := client.ListActors(ctx, "ate-env") + require.NoError(t, err) + assert.Equal(t, []string{"ate-env/run-1", "ate-env/run-2"}, names(inOne)) + + all, err := client.ListActors(ctx, "") + require.NoError(t, err) + assert.Equal(t, []string{"ate-env/run-1", "ate-env/run-2", "gavel/run-3"}, names(all)) + assert.Equal(t, "running", all[0].State) + assert.WithinDuration(t, time.Now(), all[0].Created, time.Minute) +} + +func TestListWorkersReportsTheirCapacityAndWhatActorsUse(t *testing.T) { + server := substratetest.Start(t) + server.ListPageSize = 1 + workers := []substrate.Worker{ + { + Name: "pool-a-0", Pool: "pool-a", Node: "node-1", State: "active", + Capacity: substrate.Resources{CPUMillicores: 4000, MemoryBytes: 8 << 30, Actors: 10}, + Allocated: substrate.Resources{CPUMillicores: 1500, MemoryBytes: 2 << 30, Actors: 3}, + }, + { + Name: "pool-a-1", Pool: "pool-a", Node: "node-2", State: "draining", + Capacity: substrate.Resources{CPUMillicores: 4000, MemoryBytes: 8 << 30, Actors: 10}, + }, + } + for _, worker := range workers { + server.AddWorker(worker) + } + + got, err := dial(t, server).ListWorkers(testContext(t)) + require.NoError(t, err) + assert.Equal(t, workers, got) +} diff --git a/pkg/substrate/substrate.go b/pkg/substrate/substrate.go index 5229298d..471e4de4 100644 --- a/pkg/substrate/substrate.go +++ b/pkg/substrate/substrate.go @@ -177,10 +177,10 @@ func (c *Client) Create(ctx context.Context, environment Environment) (*Env, err func (e *Env) ID() string { return e.id } // Actor is the environment's SPIFFE ID, which the egress gateway names it by. -func (e *Env) Actor() string { return Actor(e.client.options.Atespace, e.id) } +func (e *Env) Actor() string { return SPIFFEID(e.client.options.Atespace, e.id) } -// Actor is the SPIFFE ID of the environment id in atespace. -func Actor(atespace, id string) string { +// SPIFFEID is the SPIFFE ID of the environment id in atespace. +func SPIFFEID(atespace, id string) string { return "spiffe://substrate-actor.local/actor/" + atespace + "/" + id } @@ -206,23 +206,15 @@ const fileChunk = 256 * 1024 // actor was created before cutoff: the ones a caller that died mid-run left behind, once no run // of theirs can still be live. It returns the IDs it deleted, and an error for each it couldn't. func (c *Client) Sweep(ctx context.Context, prefix string, cutoff time.Time) ([]string, error) { + actors, err := c.ListActors(ctx, c.options.Atespace) + if err != nil { + return nil, err + } var stale []string - request := &ateapipb.ListActorsRequest{Atespace: c.options.Atespace, Options: &ateapipb.ListOptions{PageSize: 100}} - for { - page, err := c.control.ListActors(ctx, request) - if err != nil { - return nil, fmt.Errorf("list the actors of atespace %s: %w", c.options.Atespace, err) - } - for _, actor := range page.GetActors() { - metadata := actor.GetMetadata() - if strings.HasPrefix(metadata.GetName(), prefix) && metadata.GetCreateTime() != nil && metadata.GetCreateTime().AsTime().Before(cutoff) { - stale = append(stale, metadata.GetName()) - } - } - if page.GetNextPageToken() == "" { - break + for _, actor := range actors { + if strings.HasPrefix(actor.Name, prefix) && !actor.Created.IsZero() && actor.Created.Before(cutoff) { + stale = append(stale, actor.Name) } - request.Options.PageToken = page.GetNextPageToken() } var swept []string var errs []error diff --git a/pkg/substrate/substratetest/server.go b/pkg/substrate/substratetest/server.go index 46b74303..af3903f4 100644 --- a/pkg/substrate/substratetest/server.go +++ b/pkg/substrate/substratetest/server.go @@ -49,7 +49,8 @@ type Server struct { // StreamLimit, when positive, ends every StreamProcessOutput stream after that many output // messages, as Substrate's router ends a long stream, so a client has to reconnect. StreamLimit int - // ListPageSize, when positive, caps every ListActors page, so a client has to page. + // ListPageSize, when positive, caps every ListActors and ListWorkers page, so a client has to + // page. ListPageSize int envAddr, controlAddr string @@ -59,6 +60,7 @@ type Server struct { envs map[string]*Environment policies map[string]*ateapipb.EgressPolicy deleted []string + workers []*ateapipb.Worker } // Environment is one environment the server created. @@ -124,6 +126,22 @@ func (s *Server) Policy(atespace, id string) (*ateapipb.EgressPolicy, bool) { return policy, ok } +// AddWorker adds worker to the workers the control plane lists. +func (s *Server) AddWorker(worker substrate.Worker) { + s.mu.Lock() + defer s.mu.Unlock() + resources := func(r substrate.Resources) *ateapipb.WorkerResources { + return &ateapipb.WorkerResources{CpuMillicores: r.CPUMillicores, MemoryBytes: r.MemoryBytes, Actors: r.Actors} + } + s.workers = append(s.workers, &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: worker.Name}, WorkerPool: worker.Pool, NodeName: worker.Node, + Status: &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState(ateapipb.WorkerState_value["WORKER_STATE_"+strings.ToUpper(worker.State)]), + Capacity: resources(worker.Capacity), Allocated: resources(worker.Allocated), + }, + }) +} + // Deleted lists the environments deleted so far, as atespace/id. func (s *Server) Deleted() []string { s.mu.Lock() @@ -224,43 +242,71 @@ func (c control) CreateActorEgressPolicy(_ context.Context, req *ateapipb.Create return req.GetEgressPolicy(), nil } -// ListActors lists the actors of the atespace's environments, in name order, a page at a time: -// the page token is the index of the page's first actor. +// ListActors lists the actors of the atespace's environments, or of every environment when the +// request names no atespace, in atespace and name order, a page at a time. Every actor is running. func (c control) ListActors(_ context.Context, req *ateapipb.ListActorsRequest) (*ateapipb.ListActorsResponse, error) { s := c.s s.mu.Lock() defer s.mu.Unlock() var envs []*Environment for _, env := range s.envs { - if env.Atespace == req.GetAtespace() { + if req.GetAtespace() == "" || env.Atespace == req.GetAtespace() { envs = append(envs, env) } } - slices.SortFunc(envs, func(a, b *Environment) int { return strings.Compare(a.ID, b.ID) }) - start := 0 - if token := req.GetOptions().GetPageToken(); token != "" { - var err error - if start, err = strconv.Atoi(token); err != nil || start < 0 || start > len(envs) { - return nil, status.Errorf(codes.InvalidArgument, "page token %q", token) + slices.SortFunc(envs, func(a, b *Environment) int { + return strings.Compare(a.Atespace+"/"+a.ID, b.Atespace+"/"+b.ID) + }) + start, end, next, err := s.page(req.GetOptions(), len(envs)) + if err != nil { + return nil, err + } + response := &ateapipb.ListActorsResponse{NextPageToken: next} + for _, env := range envs[start:end] { + response.Actors = append(response.Actors, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: env.Atespace, Name: env.ID, CreateTime: timestamppb.New(env.Created)}, + Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING}, + }) + } + return response, nil +} + +// ListWorkers lists the workers added with AddWorker, in the order they were added, a page at a +// time. +func (c control) ListWorkers(_ context.Context, req *ateapipb.ListWorkersRequest) (*ateapipb.ListWorkersResponse, error) { + s := c.s + s.mu.Lock() + defer s.mu.Unlock() + start, end, next, err := s.page(req.GetOptions(), len(s.workers)) + if err != nil { + return nil, err + } + response := &ateapipb.ListWorkersResponse{NextPageToken: next} + for _, worker := range s.workers[start:end] { + response.Workers = append(response.Workers, proto.Clone(worker).(*ateapipb.Worker)) + } + return response, nil +} + +// page is the range of a list of n items that options' page covers, and the token of the page +// after it: a page token is the index of the page's first item. +func (s *Server) page(options *ateapipb.ListOptions, n int) (start, end int, next string, err error) { + if token := options.GetPageToken(); token != "" { + if start, err = strconv.Atoi(token); err != nil || start < 0 || start > n { + return 0, 0, "", status.Errorf(codes.InvalidArgument, "page token %q", token) } } - end := len(envs) - if size := int(req.GetOptions().GetPageSize()); size > 0 && start+size < end { + end = n + if size := int(options.GetPageSize()); size > 0 && start+size < end { end = start + size } if s.ListPageSize > 0 && start+s.ListPageSize < end { end = start + s.ListPageSize } - response := &ateapipb.ListActorsResponse{} - for _, env := range envs[start:end] { - response.Actors = append(response.Actors, &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{ - Atespace: env.Atespace, Name: env.ID, CreateTime: timestamppb.New(env.Created), - }}) - } - if end < len(envs) { - response.NextPageToken = strconv.Itoa(end) + if end < n { + next = strconv.Itoa(end) } - return response, nil + return start, end, next, nil } // process is one process started in an environment, with its output spooled. From 12fc452fe0304d8667d2ec3c38a8b2dfcb5ec800 Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Tue, 10 Nov 2026 20:17:23 +0530 Subject: [PATCH 2/8] feat(database): export ModelCallCostUSD, the per-call cost model usage and budgets total An embedding application that breaks usage down by day, tenant or model prices each call the way captain does without copying the formula. Model usage and budget spend both total it. Budget spend gains specs: a group's calls since a time, and refusing calls outside USD. --- pkg/database/budget_spend_integration_test.go | 59 ++++++++++++++++ pkg/database/budget_store.go | 2 +- pkg/database/model_usage.go | 21 +++--- pkg/database/model_usage_integration_test.go | 68 +++++++++++++------ 4 files changed, 119 insertions(+), 31 deletions(-) create mode 100644 pkg/database/budget_spend_integration_test.go diff --git a/pkg/database/budget_spend_integration_test.go b/pkg/database/budget_spend_integration_test.go new file mode 100644 index 00000000..47971059 --- /dev/null +++ b/pkg/database/budget_spend_integration_test.go @@ -0,0 +1,59 @@ +// Specs for a budget rule's spend from the model-call ledger: the calls attributed to one of its +// groups since a time, priced by ModelCallCostUSD, and refused when any of them isn't in USD. +package database + +import ( + "context" + "time" + + "github.com/flanksource/captain/pkg/api" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Budget spend", func() { + const schema = "agent_budget_spend_context" + var ( + db *DB + rule *BudgetRule + ) + + BeforeEach(func(ctx SpecContext) { + db = openSchemaScopedDatabases(ctx, "captain_budget_spend", schema)[0] + var err error + rule, err = db.CreateBudgetRule(ctx, BudgetRuleInput{Name: "per org", GroupBy: []string{"org"}, Amount: 10, Window: "now-30d"}) + Expect(err).NotTo(HaveOccurred()) + }) + + attribute := func(ctx context.Context, callID uuid.UUID, org string) { + var turnID string + Expect(db.Gorm().Table(schema+".captain_model_calls").Where("id = ?", callID).Select("turn_id").Scan(&turnID).Error).To(Succeed()) + Expect(db.SetModelCallBudgets(ctx, uuid.MustParse(turnID), callID, map[string]string{"org": org}, + []BudgetAttribution{{RuleID: rule.ID, GroupValues: map[string]string{"org": org}}})).To(Succeed()) + } + + It("totals the calls attributed to a group since a time", func(ctx SpecContext) { + since := time.Now().Add(-time.Hour) + attribute(ctx, recordModelCall(ctx, db, "acme provider priced", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 1}, 0.04), "acme") + attribute(ctx, recordModelCall(ctx, db, "acme list priced", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 0.01, OutputCost: 0.02}, 0), "acme") + attribute(ctx, recordModelCall(ctx, db, "globex", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 5}, 0), "globex") + + spend, err := db.BudgetSpendUSD(ctx, rule.ID, map[string]string{"org": "acme"}, since) + Expect(err).NotTo(HaveOccurred()) + Expect(spend).To(BeNumerically("~", 0.07, 0.000000001)) + + later, err := db.BudgetSpendUSD(ctx, rule.ID, map[string]string{"org": "acme"}, time.Now().Add(time.Hour)) + Expect(err).NotTo(HaveOccurred()) + Expect(later).To(BeZero()) + }) + + It("refuses a group holding a call outside USD", func(ctx SpecContext) { + inEuros := recordModelCall(ctx, db, "acme in euros", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 5}, 0) + Expect(db.Gorm().Table(schema+".captain_model_calls").Where("id = ?", inEuros).Update("currency", "EUR").Error).To(Succeed()) + attribute(ctx, inEuros, "acme") + + _, err := db.BudgetSpendUSD(ctx, rule.ID, map[string]string{"org": "acme"}, time.Now().Add(-time.Hour)) + Expect(err).To(MatchError("budget ledger contains 1 completed non-USD model calls")) + }) +}) diff --git a/pkg/database/budget_store.go b/pkg/database/budget_store.go index 68fa0fbf..0945c595 100644 --- a/pkg/database/budget_store.go +++ b/pkg/database/budget_store.go @@ -275,7 +275,7 @@ func (db *DB) BudgetSpendUSD(ctx context.Context, ruleID uuid.UUID, groupValues } var total float64 if err := db.budgetSpendStatement(ctx, ruleID, groupJSON, since). - Select(`COALESCE(SUM(CASE WHEN calls.provider_cost_usd > 0 THEN calls.provider_cost_usd ELSE calls.input_cost + calls.output_cost + calls.reasoning_cost + calls.cache_read_cost + calls.cache_write_cost END), 0)`). + Select("COALESCE(SUM(" + ModelCallCostUSD + "), 0)"). Scan(&total).Error; err != nil { return 0, fmt.Errorf("aggregate completed budget spend: %w", err) } diff --git a/pkg/database/model_usage.go b/pkg/database/model_usage.go index 99e55353..07eb6e29 100644 --- a/pkg/database/model_usage.go +++ b/pkg/database/model_usage.go @@ -32,17 +32,20 @@ type ModelUsageQuery struct { Schemas []string } -// modelUsageTotalsSelect mirrors api.Cost.Total() and the captain_session_costs -// view: prefer the provider-reported cost per call, with the list-price bucket -// sum as the fallback. Decided per row, not per group, so a mix of list-priced -// and provider-reported calls still totals. Non-USD calls contribute tokens but -// no cost — there is no rate table to convert them, and adding EUR to USD would -// be worse than reporting less. -const modelUsageTotalsSelect = `COALESCE(SUM(calls.input_tokens + calls.output_tokens + calls.reasoning_tokens + calls.cache_read_tokens + calls.cache_write_tokens), 0) AS total_tokens, - COALESCE(SUM(CASE WHEN upper(calls.currency) = 'USD' THEN +// ModelCallCostUSD is SQL for the USD cost of one row of captain_model_calls aliased calls. It +// mirrors api.Cost.Total() and the captain_session_costs view: the provider-reported cost when +// there is one, else the sum of the list-price buckets. Decided per row, not per group, so a mix +// of list-priced and provider-reported calls still totals. Non-USD calls cost nothing here: there +// is no rate table to convert them, and adding EUR to USD would be worse than reporting less. +const ModelCallCostUSD = `CASE WHEN upper(calls.currency) = 'USD' THEN CASE WHEN calls.provider_cost_usd > 0 THEN calls.provider_cost_usd ELSE calls.input_cost + calls.output_cost + calls.reasoning_cost + calls.cache_read_cost + calls.cache_write_cost END - ELSE 0 END), 0) AS total_cost_usd` + ELSE 0 END` + +// modelUsageTotalsSelect totals the selected calls' tokens and their ModelCallCostUSD. Non-USD +// calls contribute tokens but no cost. +const modelUsageTotalsSelect = `COALESCE(SUM(calls.input_tokens + calls.output_tokens + calls.reasoning_tokens + calls.cache_read_tokens + calls.cache_write_tokens), 0) AS total_tokens, + COALESCE(SUM(` + ModelCallCostUSD + `), 0) AS total_cost_usd` // ModelUsage aggregates Captain model calls in the selected schemas. // Configured schemas that have never been migrated have no usage and are diff --git a/pkg/database/model_usage_integration_test.go b/pkg/database/model_usage_integration_test.go index b99120ab..eef5bc65 100644 --- a/pkg/database/model_usage_integration_test.go +++ b/pkg/database/model_usage_integration_test.go @@ -1,6 +1,7 @@ package database import ( + "context" "time" "github.com/flanksource/captain/pkg/api" @@ -9,6 +10,31 @@ import ( . "github.com/onsi/gomega" ) +// recordModelCall records one finished model call in a session titled title, with usage, its +// list-price cost, and the cost its provider reported, and returns the call's ID. +func recordModelCall(ctx context.Context, db *DB, title string, usage api.Usage, cost *api.Cost, providerCostUSD float64) uuid.UUID { + session, err := db.CreateOrGetSession(ctx, CreateSessionInput{ + ProviderSessionID: title, Source: "codex", Provider: "openai", + HostID: "usage-scope-test", Title: title, + }) + Expect(err).NotTo(HaveOccurred()) + turn, _, err := db.CreateChatTurn(ctx, CreateChatTurnInput{ + SessionID: session.ID, ProviderTurnID: title + " turn", + }) + Expect(err).NotTo(HaveOccurred()) + run, err := db.CreatePromptRun(ctx, CreatePromptRunInput{SessionID: session.ID, TurnID: &turn.ID}) + Expect(err).NotTo(HaveOccurred()) + callID, err := db.CreateChatModelCall(ctx, CreateChatModelCallInput{ + TurnID: turn.ID, PromptRunID: run.ID, Model: "model-usage-scope", Provider: "openai", Mode: "cli", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(db.FinishChatModelCall(ctx, FinishChatModelCallInput{ + ID: callID, Status: ModelCallStatusSucceeded, StopReason: "end_turn", + Event: api.Event{Usage: &usage, CostUSD: providerCostUSD}, Cost: cost, + })).To(Succeed()) + return callID +} + var _ = Describe("Model usage aggregation", func() { It("counts only the sessions an owner's subquery selects", func(ctx SpecContext) { const ( @@ -25,27 +51,8 @@ var _ = Describe("Model usage aggregation", func() { db := openSchemaScopedDatabases(ctx, "captain_model_usage", "agent_usage_scope_context")[0] since := time.Now().Add(-time.Hour) - record := func(title string, usage api.Usage, providerCostUSD float64) uuid.UUID { - session, err := db.CreateOrGetSession(ctx, CreateSessionInput{ - ProviderSessionID: title, Source: "codex", Provider: "openai", - HostID: "usage-scope-test", Title: title, - }) - Expect(err).NotTo(HaveOccurred()) - turn, _, err := db.CreateChatTurn(ctx, CreateChatTurnInput{ - SessionID: session.ID, ProviderTurnID: title + " turn", - }) - Expect(err).NotTo(HaveOccurred()) - run, err := db.CreatePromptRun(ctx, CreatePromptRunInput{SessionID: session.ID, TurnID: &turn.ID}) - Expect(err).NotTo(HaveOccurred()) - callID, err := db.CreateChatModelCall(ctx, CreateChatModelCallInput{ - TurnID: turn.ID, PromptRunID: run.ID, Model: "model-usage-scope", Provider: "openai", Mode: "cli", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(db.FinishChatModelCall(ctx, FinishChatModelCallInput{ - ID: callID, Status: ModelCallStatusSucceeded, StopReason: "end_turn", - Event: api.Event{Usage: &usage, CostUSD: providerCostUSD}, Cost: &api.Cost{}, - })).To(Succeed()) - return session.ID + record := func(title string, usage api.Usage, providerCostUSD float64) { + recordModelCall(ctx, db, title, usage, &api.Cost{}, providerCostUSD) } record(tenantTitle, api.Usage{InputTokens: tenantInputTokens, OutputTokens: tenantOutputToken}, tenantProviderCostUSD) @@ -70,4 +77,23 @@ var _ = Describe("Model usage aggregation", func() { Expect(unscoped.TotalTokens).To(Equal(tenantInputTokens + tenantOutputToken + 1000)) Expect(unscoped.TotalCostUSD).To(BeNumerically("~", tenantProviderCostUSD+otherProviderCostUSD, 0.000000001)) }) + + It("prices each call by ModelCallCostUSD: its provider's cost, else its list price, and nothing outside USD", func(ctx SpecContext) { + const schema = "agent_call_cost_context" + db := openSchemaScopedDatabases(ctx, "captain_model_call_cost", schema)[0] + providerPriced := recordModelCall(ctx, db, "provider priced", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 1}, 0.04) + listPriced := recordModelCall(ctx, db, "list priced", api.Usage{InputTokens: 10, OutputTokens: 5}, &api.Cost{InputCost: 0.01, OutputCost: 0.02}, 0) + inEuros := recordModelCall(ctx, db, "in euros", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 5}, 0) + Expect(db.Gorm().Table(schema+".captain_model_calls").Where("id = ?", inEuros).Update("currency", "EUR").Error).To(Succeed()) + + cost := func(id uuid.UUID) float64 { + var usd float64 + Expect(db.Gorm().Table(schema+".captain_model_calls AS calls").Where("calls.id = ?", id). + Select(ModelCallCostUSD).Scan(&usd).Error).To(Succeed()) + return usd + } + Expect(cost(providerPriced)).To(BeNumerically("~", 0.04, 0.000000001)) + Expect(cost(listPriced)).To(BeNumerically("~", 0.03, 0.000000001)) + Expect(cost(inEuros)).To(BeZero()) + }) }) From 4a38c52cf9e50202d714094a78125e355458ff4c Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Wed, 11 Nov 2026 11:20:08 +0530 Subject: [PATCH 3/8] feat(budgets): export WindowStart, where a budget window starts So an embedding application reporting spend reads a rule's window as captain enforces it. --- pkg/budgets/types.go | 6 ++++-- pkg/budgets/window_test.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 pkg/budgets/window_test.go diff --git a/pkg/budgets/types.go b/pkg/budgets/types.go index 4d0a7175..91b32b88 100644 --- a/pkg/budgets/types.go +++ b/pkg/budgets/types.go @@ -53,7 +53,7 @@ func normalizeInput(input RuleInput) (RuleInput, error) { if input.Name == "" || input.Amount <= 0 || input.Window == "" { return RuleInput{}, fmt.Errorf("%w: name, positive amount, and window are required", ErrInvalid) } - if _, err := windowStart(input.Window, time.Now().UTC()); err != nil { + if _, err := WindowStart(input.Window, time.Now().UTC()); err != nil { return RuleInput{}, fmt.Errorf("%w: %v", ErrInvalid, err) } dimensions := make(map[string]string, len(input.Match.Dimensions)) @@ -95,7 +95,9 @@ func cleanList(values []string) []string { return out } -func windowStart(raw string, now time.Time) (time.Time, error) { +// WindowStart is where a budget window that ends at now starts: raw is datemath relative to now, +// such as now/M or now-30d, evaluated in UTC. An absolute or future start is refused. +func WindowStart(raw string, now time.Time) (time.Time, error) { raw = strings.TrimSpace(raw) if !strings.HasPrefix(raw, "now") { return time.Time{}, fmt.Errorf("budget window %q must be relative to now", raw) diff --git a/pkg/budgets/window_test.go b/pkg/budgets/window_test.go new file mode 100644 index 00000000..4e2846ec --- /dev/null +++ b/pkg/budgets/window_test.go @@ -0,0 +1,37 @@ +// Tests where a budget window starts: relative datemath evaluated in UTC, and why an absolute, +// unparseable or future window is refused. +package budgets_test + +import ( + "testing" + "time" + + "github.com/flanksource/captain/pkg/budgets" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWindowStart(t *testing.T) { + now := time.Date(2026, 11, 11, 15, 30, 0, 0, time.UTC) + + for window, want := range map[string]time.Time{ + "now/d": time.Date(2026, 11, 11, 0, 0, 0, 0, time.UTC), + "now/M": time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC), + "now-30d": now.AddDate(0, 0, -30), + " now/d ": time.Date(2026, 11, 11, 0, 0, 0, 0, time.UTC), + } { + start, err := budgets.WindowStart(window, now) + require.NoError(t, err, window) + assert.Equal(t, want, start, window) + } + + for window, reason := range map[string]string{ + "2026-11-01": `budget window "2026-11-01" must be relative to now`, + "now+1d": `budget window "now+1d" starts after now`, + "now-x": `invalid budget window "now-x": `, + } { + _, err := budgets.WindowStart(window, now) + require.Error(t, err, window) + assert.Contains(t, err.Error(), reason, window) + } +} From c793dff5b28ded9e588e1941e17f33f65178a2d2 Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Wed, 11 Nov 2026 11:27:48 +0530 Subject: [PATCH 4/8] feat(substrate): actors, workers and their resources encode as JSON --- pkg/substrate/inventory.go | 22 +++++++++++++--------- pkg/substrate/inventory_test.go | 22 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/pkg/substrate/inventory.go b/pkg/substrate/inventory.go index 90962c8d..4496ff5c 100644 --- a/pkg/substrate/inventory.go +++ b/pkg/substrate/inventory.go @@ -13,28 +13,32 @@ import ( // Actor is one actor in a substrate: an environment, or anything else Substrate runs. type Actor struct { - Atespace, Name string + Atespace string `json:"atespace"` + Name string `json:"name"` // State is resuming, running, suspending, suspended, crashed, deleting or reverting. - State string + State string `json:"state"` // Created is when the actor was created; zero when Substrate doesn't say. - Created time.Time + Created time.Time `json:"created,omitzero"` } // Worker is one worker pod that actors run on. type Worker struct { - Name, Pool, Node string + Name string `json:"name"` + Pool string `json:"pool"` + Node string `json:"node"` // State is active, draining or unavailable. - State string + State string `json:"state"` // Capacity is what the worker can hold, and Allocated what its actors are allocated of it, // their runtime's overhead included. - Capacity, Allocated Resources + Capacity Resources `json:"capacity"` + Allocated Resources `json:"allocated"` } // Resources is an amount of a worker's CPU, memory and actor slots. type Resources struct { - CPUMillicores int64 - MemoryBytes int64 - Actors int32 + CPUMillicores int64 `json:"cpuMillicores"` + MemoryBytes int64 `json:"memoryBytes"` + Actors int32 `json:"actors"` } // listPageSize is how many items one list call asks Substrate for. diff --git a/pkg/substrate/inventory_test.go b/pkg/substrate/inventory_test.go index fe4db117..3f7e9950 100644 --- a/pkg/substrate/inventory_test.go +++ b/pkg/substrate/inventory_test.go @@ -3,6 +3,7 @@ package substrate_test import ( + "encoding/json" "testing" "time" @@ -65,3 +66,24 @@ func TestListWorkersReportsTheirCapacityAndWhatActorsUse(t *testing.T) { require.NoError(t, err) assert.Equal(t, workers, got) } + +func TestInventoryEncodesAsJSON(t *testing.T) { + worker, err := json.Marshal(substrate.Worker{ + Name: "pool-a-0", Pool: "pool-a", Node: "node-1", State: "active", + Capacity: substrate.Resources{CPUMillicores: 4000, MemoryBytes: 8, Actors: 10}, + Allocated: substrate.Resources{CPUMillicores: 1500, MemoryBytes: 2, Actors: 3}, + }) + require.NoError(t, err) + assert.JSONEq(t, `{"name":"pool-a-0","pool":"pool-a","node":"node-1","state":"active", + "capacity":{"cpuMillicores":4000,"memoryBytes":8,"actors":10}, + "allocated":{"cpuMillicores":1500,"memoryBytes":2,"actors":3}}`, string(worker)) + + created := time.Date(2026, 11, 11, 10, 0, 0, 0, time.UTC) + actors, err := json.Marshal([]substrate.Actor{ + {Atespace: "ate-env", Name: "run-1", State: "running", Created: created}, + {Atespace: "ate-env", Name: "run-2", State: "crashed"}, + }) + require.NoError(t, err) + assert.JSONEq(t, `[{"atespace":"ate-env","name":"run-1","state":"running","created":"2026-11-11T10:00:00Z"}, + {"atespace":"ate-env","name":"run-2","state":"crashed"}]`, string(actors), "a zero creation time is left out") +} From db95a49bf5053c71345ce3d36adc4c5f9fdbf048 Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Wed, 11 Nov 2026 12:19:30 +0530 Subject: [PATCH 5/8] fix(database): BudgetSpendUSD counts non-USD calls and totals spend in one statement Two statements let a non-USD call complete between the count and the total, which priced it at zero instead of failing closed. --- pkg/database/budget_store.go | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/pkg/database/budget_store.go b/pkg/database/budget_store.go index 0945c595..8f3706b5 100644 --- a/pkg/database/budget_store.go +++ b/pkg/database/budget_store.go @@ -259,25 +259,24 @@ func (db *DB) budgetSpendStatement(ctx context.Context, ruleID uuid.UUID, groupJ // BudgetSpendUSD derives completed spend from the model-call ledger. Any // non-USD completed call in the group fails closed because no conversion rate -// is authoritative here. +// is authoritative here. One statement counts those calls and totals the spend, +// so a call completing between the two can't be priced at zero unnoticed. func (db *DB) BudgetSpendUSD(ctx context.Context, ruleID uuid.UUID, groupValues map[string]string, since time.Time) (float64, error) { groupJSON, err := json.Marshal(cloneStrings(groupValues)) if err != nil { return 0, fmt.Errorf("encode budget group: %w", err) } - var nonUSD int64 - if err := db.budgetSpendStatement(ctx, ruleID, groupJSON, since). - Where("upper(calls.currency) <> 'USD'").Count(&nonUSD).Error; err != nil { - return 0, fmt.Errorf("inspect budget ledger currency: %w", err) - } - if nonUSD > 0 { - return 0, fmt.Errorf("budget ledger contains %d completed non-USD model calls", nonUSD) + var spend struct { + NonUSD int64 + Total float64 } - var total float64 if err := db.budgetSpendStatement(ctx, ruleID, groupJSON, since). - Select("COALESCE(SUM(" + ModelCallCostUSD + "), 0)"). - Scan(&total).Error; err != nil { + Select("count(*) FILTER (WHERE upper(calls.currency) <> 'USD') AS non_usd, COALESCE(SUM(" + ModelCallCostUSD + "), 0) AS total"). + Scan(&spend).Error; err != nil { return 0, fmt.Errorf("aggregate completed budget spend: %w", err) } - return total, nil + if spend.NonUSD > 0 { + return 0, fmt.Errorf("budget ledger contains %d completed non-USD model calls", spend.NonUSD) + } + return spend.Total, nil } From 27fabe74803314fe925ce45a768a8233623b9de9 Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Wed, 11 Nov 2026 12:19:30 +0530 Subject: [PATCH 6/8] fix(substratetest): list every atespace's actors in atespace, then name, order Sorting the joined atespace/name put atespace a-b before a. --- pkg/substrate/substratetest/server.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/substrate/substratetest/server.go b/pkg/substrate/substratetest/server.go index af3903f4..37117018 100644 --- a/pkg/substrate/substratetest/server.go +++ b/pkg/substrate/substratetest/server.go @@ -3,6 +3,7 @@ package substratetest import ( + "cmp" "context" "crypto/ecdsa" "crypto/elliptic" @@ -255,7 +256,7 @@ func (c control) ListActors(_ context.Context, req *ateapipb.ListActorsRequest) } } slices.SortFunc(envs, func(a, b *Environment) int { - return strings.Compare(a.Atespace+"/"+a.ID, b.Atespace+"/"+b.ID) + return cmp.Or(strings.Compare(a.Atespace, b.Atespace), strings.Compare(a.ID, b.ID)) }) start, end, next, err := s.page(req.GetOptions(), len(envs)) if err != nil { From 67ac285694d6f5e28801298040b33148d4f5ebe7 Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Wed, 11 Nov 2026 12:19:30 +0530 Subject: [PATCH 7/8] docs(substrate): what an actor's or worker's state is when Substrate gives none or a new one --- pkg/substrate/inventory.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/substrate/inventory.go b/pkg/substrate/inventory.go index 4496ff5c..6bebd983 100644 --- a/pkg/substrate/inventory.go +++ b/pkg/substrate/inventory.go @@ -15,18 +15,21 @@ import ( type Actor struct { Atespace string `json:"atespace"` Name string `json:"name"` - // State is resuming, running, suspending, suspended, crashed, deleting or reverting. + // State is resuming, running, suspending, suspended, crashed, deleting or reverting; + // unspecified when Substrate doesn't say, or the state's number for one this client doesn't know. State string `json:"state"` // Created is when the actor was created; zero when Substrate doesn't say. Created time.Time `json:"created,omitzero"` } -// Worker is one worker pod that actors run on. +// Worker is one of substrate's workers, the pods that actors run on. Name is Substrate's name for +// it. type Worker struct { Name string `json:"name"` Pool string `json:"pool"` Node string `json:"node"` - // State is active, draining or unavailable. + // State is active, draining or unavailable; unspecified when the worker hasn't reported one, + // or the state's number for one this client doesn't know. State string `json:"state"` // Capacity is what the worker can hold, and Allocated what its actors are allocated of it, // their runtime's overhead included. From 440799b20dd0b42301274e1b4de9b8696c55cc48 Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Wed, 11 Nov 2026 12:19:31 +0530 Subject: [PATCH 8/8] test(database): call recordModelCall directly in the tenant-scope usage spec --- pkg/database/model_usage_integration_test.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/database/model_usage_integration_test.go b/pkg/database/model_usage_integration_test.go index eef5bc65..f6d2ac25 100644 --- a/pkg/database/model_usage_integration_test.go +++ b/pkg/database/model_usage_integration_test.go @@ -51,12 +51,8 @@ var _ = Describe("Model usage aggregation", func() { db := openSchemaScopedDatabases(ctx, "captain_model_usage", "agent_usage_scope_context")[0] since := time.Now().Add(-time.Hour) - record := func(title string, usage api.Usage, providerCostUSD float64) { - recordModelCall(ctx, db, title, usage, &api.Cost{}, providerCostUSD) - } - - record(tenantTitle, api.Usage{InputTokens: tenantInputTokens, OutputTokens: tenantOutputToken}, tenantProviderCostUSD) - record(otherTenantTitle, api.Usage{InputTokens: 500, OutputTokens: 500}, otherProviderCostUSD) + recordModelCall(ctx, db, tenantTitle, api.Usage{InputTokens: tenantInputTokens, OutputTokens: tenantOutputToken}, &api.Cost{}, tenantProviderCostUSD) + recordModelCall(ctx, db, otherTenantTitle, api.Usage{InputTokens: 500, OutputTokens: 500}, &api.Cost{}, otherProviderCostUSD) // Stands in for an embedding application's ownership table: it knows // which sessions belong to the tenant, Captain knows what they cost.