diff --git a/pkg/budgets/types.go b/pkg/budgets/types.go index 4d0a7175..91b32b88 100644 --- a/pkg/budgets/types.go +++ b/pkg/budgets/types.go @@ -53,7 +53,7 @@ func normalizeInput(input RuleInput) (RuleInput, error) { if input.Name == "" || input.Amount <= 0 || input.Window == "" { return RuleInput{}, fmt.Errorf("%w: name, positive amount, and window are required", ErrInvalid) } - if _, err := windowStart(input.Window, time.Now().UTC()); err != nil { + if _, err := WindowStart(input.Window, time.Now().UTC()); err != nil { return RuleInput{}, fmt.Errorf("%w: %v", ErrInvalid, err) } dimensions := make(map[string]string, len(input.Match.Dimensions)) @@ -95,7 +95,9 @@ func cleanList(values []string) []string { return out } -func windowStart(raw string, now time.Time) (time.Time, error) { +// WindowStart is where a budget window that ends at now starts: raw is datemath relative to now, +// such as now/M or now-30d, evaluated in UTC. An absolute or future start is refused. +func WindowStart(raw string, now time.Time) (time.Time, error) { raw = strings.TrimSpace(raw) if !strings.HasPrefix(raw, "now") { return time.Time{}, fmt.Errorf("budget window %q must be relative to now", raw) diff --git a/pkg/budgets/window_test.go b/pkg/budgets/window_test.go new file mode 100644 index 00000000..4e2846ec --- /dev/null +++ b/pkg/budgets/window_test.go @@ -0,0 +1,37 @@ +// Tests where a budget window starts: relative datemath evaluated in UTC, and why an absolute, +// unparseable or future window is refused. +package budgets_test + +import ( + "testing" + "time" + + "github.com/flanksource/captain/pkg/budgets" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWindowStart(t *testing.T) { + now := time.Date(2026, 11, 11, 15, 30, 0, 0, time.UTC) + + for window, want := range map[string]time.Time{ + "now/d": time.Date(2026, 11, 11, 0, 0, 0, 0, time.UTC), + "now/M": time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC), + "now-30d": now.AddDate(0, 0, -30), + " now/d ": time.Date(2026, 11, 11, 0, 0, 0, 0, time.UTC), + } { + start, err := budgets.WindowStart(window, now) + require.NoError(t, err, window) + assert.Equal(t, want, start, window) + } + + for window, reason := range map[string]string{ + "2026-11-01": `budget window "2026-11-01" must be relative to now`, + "now+1d": `budget window "now+1d" starts after now`, + "now-x": `invalid budget window "now-x": `, + } { + _, err := budgets.WindowStart(window, now) + require.Error(t, err, window) + assert.Contains(t, err.Error(), reason, window) + } +} diff --git a/pkg/database/budget_spend_integration_test.go b/pkg/database/budget_spend_integration_test.go new file mode 100644 index 00000000..47971059 --- /dev/null +++ b/pkg/database/budget_spend_integration_test.go @@ -0,0 +1,59 @@ +// Specs for a budget rule's spend from the model-call ledger: the calls attributed to one of its +// groups since a time, priced by ModelCallCostUSD, and refused when any of them isn't in USD. +package database + +import ( + "context" + "time" + + "github.com/flanksource/captain/pkg/api" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Budget spend", func() { + const schema = "agent_budget_spend_context" + var ( + db *DB + rule *BudgetRule + ) + + BeforeEach(func(ctx SpecContext) { + db = openSchemaScopedDatabases(ctx, "captain_budget_spend", schema)[0] + var err error + rule, err = db.CreateBudgetRule(ctx, BudgetRuleInput{Name: "per org", GroupBy: []string{"org"}, Amount: 10, Window: "now-30d"}) + Expect(err).NotTo(HaveOccurred()) + }) + + attribute := func(ctx context.Context, callID uuid.UUID, org string) { + var turnID string + Expect(db.Gorm().Table(schema+".captain_model_calls").Where("id = ?", callID).Select("turn_id").Scan(&turnID).Error).To(Succeed()) + Expect(db.SetModelCallBudgets(ctx, uuid.MustParse(turnID), callID, map[string]string{"org": org}, + []BudgetAttribution{{RuleID: rule.ID, GroupValues: map[string]string{"org": org}}})).To(Succeed()) + } + + It("totals the calls attributed to a group since a time", func(ctx SpecContext) { + since := time.Now().Add(-time.Hour) + attribute(ctx, recordModelCall(ctx, db, "acme provider priced", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 1}, 0.04), "acme") + attribute(ctx, recordModelCall(ctx, db, "acme list priced", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 0.01, OutputCost: 0.02}, 0), "acme") + attribute(ctx, recordModelCall(ctx, db, "globex", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 5}, 0), "globex") + + spend, err := db.BudgetSpendUSD(ctx, rule.ID, map[string]string{"org": "acme"}, since) + Expect(err).NotTo(HaveOccurred()) + Expect(spend).To(BeNumerically("~", 0.07, 0.000000001)) + + later, err := db.BudgetSpendUSD(ctx, rule.ID, map[string]string{"org": "acme"}, time.Now().Add(time.Hour)) + Expect(err).NotTo(HaveOccurred()) + Expect(later).To(BeZero()) + }) + + It("refuses a group holding a call outside USD", func(ctx SpecContext) { + inEuros := recordModelCall(ctx, db, "acme in euros", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 5}, 0) + Expect(db.Gorm().Table(schema+".captain_model_calls").Where("id = ?", inEuros).Update("currency", "EUR").Error).To(Succeed()) + attribute(ctx, inEuros, "acme") + + _, err := db.BudgetSpendUSD(ctx, rule.ID, map[string]string{"org": "acme"}, time.Now().Add(-time.Hour)) + Expect(err).To(MatchError("budget ledger contains 1 completed non-USD model calls")) + }) +}) diff --git a/pkg/database/budget_store.go b/pkg/database/budget_store.go index 68fa0fbf..8f3706b5 100644 --- a/pkg/database/budget_store.go +++ b/pkg/database/budget_store.go @@ -259,25 +259,24 @@ func (db *DB) budgetSpendStatement(ctx context.Context, ruleID uuid.UUID, groupJ // BudgetSpendUSD derives completed spend from the model-call ledger. Any // non-USD completed call in the group fails closed because no conversion rate -// is authoritative here. +// is authoritative here. One statement counts those calls and totals the spend, +// so a call completing between the two can't be priced at zero unnoticed. func (db *DB) BudgetSpendUSD(ctx context.Context, ruleID uuid.UUID, groupValues map[string]string, since time.Time) (float64, error) { groupJSON, err := json.Marshal(cloneStrings(groupValues)) if err != nil { return 0, fmt.Errorf("encode budget group: %w", err) } - var nonUSD int64 - if err := db.budgetSpendStatement(ctx, ruleID, groupJSON, since). - Where("upper(calls.currency) <> 'USD'").Count(&nonUSD).Error; err != nil { - return 0, fmt.Errorf("inspect budget ledger currency: %w", err) - } - if nonUSD > 0 { - return 0, fmt.Errorf("budget ledger contains %d completed non-USD model calls", nonUSD) + var spend struct { + NonUSD int64 + Total float64 } - var total float64 if err := db.budgetSpendStatement(ctx, ruleID, groupJSON, since). - Select(`COALESCE(SUM(CASE WHEN calls.provider_cost_usd > 0 THEN calls.provider_cost_usd ELSE calls.input_cost + calls.output_cost + calls.reasoning_cost + calls.cache_read_cost + calls.cache_write_cost END), 0)`). - Scan(&total).Error; err != nil { + Select("count(*) FILTER (WHERE upper(calls.currency) <> 'USD') AS non_usd, COALESCE(SUM(" + ModelCallCostUSD + "), 0) AS total"). + Scan(&spend).Error; err != nil { return 0, fmt.Errorf("aggregate completed budget spend: %w", err) } - return total, nil + if spend.NonUSD > 0 { + return 0, fmt.Errorf("budget ledger contains %d completed non-USD model calls", spend.NonUSD) + } + return spend.Total, nil } diff --git a/pkg/database/model_usage.go b/pkg/database/model_usage.go index 99e55353..07eb6e29 100644 --- a/pkg/database/model_usage.go +++ b/pkg/database/model_usage.go @@ -32,17 +32,20 @@ type ModelUsageQuery struct { Schemas []string } -// modelUsageTotalsSelect mirrors api.Cost.Total() and the captain_session_costs -// view: prefer the provider-reported cost per call, with the list-price bucket -// sum as the fallback. Decided per row, not per group, so a mix of list-priced -// and provider-reported calls still totals. Non-USD calls contribute tokens but -// no cost — there is no rate table to convert them, and adding EUR to USD would -// be worse than reporting less. -const modelUsageTotalsSelect = `COALESCE(SUM(calls.input_tokens + calls.output_tokens + calls.reasoning_tokens + calls.cache_read_tokens + calls.cache_write_tokens), 0) AS total_tokens, - COALESCE(SUM(CASE WHEN upper(calls.currency) = 'USD' THEN +// ModelCallCostUSD is SQL for the USD cost of one row of captain_model_calls aliased calls. It +// mirrors api.Cost.Total() and the captain_session_costs view: the provider-reported cost when +// there is one, else the sum of the list-price buckets. Decided per row, not per group, so a mix +// of list-priced and provider-reported calls still totals. Non-USD calls cost nothing here: there +// is no rate table to convert them, and adding EUR to USD would be worse than reporting less. +const ModelCallCostUSD = `CASE WHEN upper(calls.currency) = 'USD' THEN CASE WHEN calls.provider_cost_usd > 0 THEN calls.provider_cost_usd ELSE calls.input_cost + calls.output_cost + calls.reasoning_cost + calls.cache_read_cost + calls.cache_write_cost END - ELSE 0 END), 0) AS total_cost_usd` + ELSE 0 END` + +// modelUsageTotalsSelect totals the selected calls' tokens and their ModelCallCostUSD. Non-USD +// calls contribute tokens but no cost. +const modelUsageTotalsSelect = `COALESCE(SUM(calls.input_tokens + calls.output_tokens + calls.reasoning_tokens + calls.cache_read_tokens + calls.cache_write_tokens), 0) AS total_tokens, + COALESCE(SUM(` + ModelCallCostUSD + `), 0) AS total_cost_usd` // ModelUsage aggregates Captain model calls in the selected schemas. // Configured schemas that have never been migrated have no usage and are diff --git a/pkg/database/model_usage_integration_test.go b/pkg/database/model_usage_integration_test.go index b99120ab..f6d2ac25 100644 --- a/pkg/database/model_usage_integration_test.go +++ b/pkg/database/model_usage_integration_test.go @@ -1,6 +1,7 @@ package database import ( + "context" "time" "github.com/flanksource/captain/pkg/api" @@ -9,6 +10,31 @@ import ( . "github.com/onsi/gomega" ) +// recordModelCall records one finished model call in a session titled title, with usage, its +// list-price cost, and the cost its provider reported, and returns the call's ID. +func recordModelCall(ctx context.Context, db *DB, title string, usage api.Usage, cost *api.Cost, providerCostUSD float64) uuid.UUID { + session, err := db.CreateOrGetSession(ctx, CreateSessionInput{ + ProviderSessionID: title, Source: "codex", Provider: "openai", + HostID: "usage-scope-test", Title: title, + }) + Expect(err).NotTo(HaveOccurred()) + turn, _, err := db.CreateChatTurn(ctx, CreateChatTurnInput{ + SessionID: session.ID, ProviderTurnID: title + " turn", + }) + Expect(err).NotTo(HaveOccurred()) + run, err := db.CreatePromptRun(ctx, CreatePromptRunInput{SessionID: session.ID, TurnID: &turn.ID}) + Expect(err).NotTo(HaveOccurred()) + callID, err := db.CreateChatModelCall(ctx, CreateChatModelCallInput{ + TurnID: turn.ID, PromptRunID: run.ID, Model: "model-usage-scope", Provider: "openai", Mode: "cli", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(db.FinishChatModelCall(ctx, FinishChatModelCallInput{ + ID: callID, Status: ModelCallStatusSucceeded, StopReason: "end_turn", + Event: api.Event{Usage: &usage, CostUSD: providerCostUSD}, Cost: cost, + })).To(Succeed()) + return callID +} + var _ = Describe("Model usage aggregation", func() { It("counts only the sessions an owner's subquery selects", func(ctx SpecContext) { const ( @@ -25,31 +51,8 @@ var _ = Describe("Model usage aggregation", func() { db := openSchemaScopedDatabases(ctx, "captain_model_usage", "agent_usage_scope_context")[0] since := time.Now().Add(-time.Hour) - record := func(title string, usage api.Usage, providerCostUSD float64) uuid.UUID { - session, err := db.CreateOrGetSession(ctx, CreateSessionInput{ - ProviderSessionID: title, Source: "codex", Provider: "openai", - HostID: "usage-scope-test", Title: title, - }) - Expect(err).NotTo(HaveOccurred()) - turn, _, err := db.CreateChatTurn(ctx, CreateChatTurnInput{ - SessionID: session.ID, ProviderTurnID: title + " turn", - }) - Expect(err).NotTo(HaveOccurred()) - run, err := db.CreatePromptRun(ctx, CreatePromptRunInput{SessionID: session.ID, TurnID: &turn.ID}) - Expect(err).NotTo(HaveOccurred()) - callID, err := db.CreateChatModelCall(ctx, CreateChatModelCallInput{ - TurnID: turn.ID, PromptRunID: run.ID, Model: "model-usage-scope", Provider: "openai", Mode: "cli", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(db.FinishChatModelCall(ctx, FinishChatModelCallInput{ - ID: callID, Status: ModelCallStatusSucceeded, StopReason: "end_turn", - Event: api.Event{Usage: &usage, CostUSD: providerCostUSD}, Cost: &api.Cost{}, - })).To(Succeed()) - return session.ID - } - - record(tenantTitle, api.Usage{InputTokens: tenantInputTokens, OutputTokens: tenantOutputToken}, tenantProviderCostUSD) - record(otherTenantTitle, api.Usage{InputTokens: 500, OutputTokens: 500}, otherProviderCostUSD) + recordModelCall(ctx, db, tenantTitle, api.Usage{InputTokens: tenantInputTokens, OutputTokens: tenantOutputToken}, &api.Cost{}, tenantProviderCostUSD) + recordModelCall(ctx, db, otherTenantTitle, api.Usage{InputTokens: 500, OutputTokens: 500}, &api.Cost{}, otherProviderCostUSD) // Stands in for an embedding application's ownership table: it knows // which sessions belong to the tenant, Captain knows what they cost. @@ -70,4 +73,23 @@ var _ = Describe("Model usage aggregation", func() { Expect(unscoped.TotalTokens).To(Equal(tenantInputTokens + tenantOutputToken + 1000)) Expect(unscoped.TotalCostUSD).To(BeNumerically("~", tenantProviderCostUSD+otherProviderCostUSD, 0.000000001)) }) + + It("prices each call by ModelCallCostUSD: its provider's cost, else its list price, and nothing outside USD", func(ctx SpecContext) { + const schema = "agent_call_cost_context" + db := openSchemaScopedDatabases(ctx, "captain_model_call_cost", schema)[0] + providerPriced := recordModelCall(ctx, db, "provider priced", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 1}, 0.04) + listPriced := recordModelCall(ctx, db, "list priced", api.Usage{InputTokens: 10, OutputTokens: 5}, &api.Cost{InputCost: 0.01, OutputCost: 0.02}, 0) + inEuros := recordModelCall(ctx, db, "in euros", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 5}, 0) + Expect(db.Gorm().Table(schema+".captain_model_calls").Where("id = ?", inEuros).Update("currency", "EUR").Error).To(Succeed()) + + cost := func(id uuid.UUID) float64 { + var usd float64 + Expect(db.Gorm().Table(schema+".captain_model_calls AS calls").Where("calls.id = ?", id). + Select(ModelCallCostUSD).Scan(&usd).Error).To(Succeed()) + return usd + } + Expect(cost(providerPriced)).To(BeNumerically("~", 0.04, 0.000000001)) + Expect(cost(listPriced)).To(BeNumerically("~", 0.03, 0.000000001)) + Expect(cost(inEuros)).To(BeZero()) + }) }) diff --git a/pkg/substrate/README.md b/pkg/substrate/README.md index eee40c65..40f6a0fe 100644 --- a/pkg/substrate/README.md +++ b/pkg/substrate/README.md @@ -27,6 +27,11 @@ An `Env` moves files in and out (`WriteFile`, `ReadFile`, both streamed) and sta stdin of a process started with `Stdin` (`Input`, one call per write, so an ended stream loses nothing). It waits for its exit code (`Wait`) and kills its process group (`Kill`). `Delete` deletes the environment. +`Client.ListActors(atespace)` lists the actors of an atespace, or of every atespace when it is +empty, with their state and create time. `Client.ListWorkers()` lists the workers they run on, with +each one's pool, node, state, and capacity and allocation (CPU, memory and actor slots). `Sweep` +deletes the environments a dead caller left behind, from the atespace's actor list. + ## Generated API `internal/ateapi`, `internal/credprovider` and `internal/ateenvv1alpha` are generated by `go diff --git a/pkg/substrate/e2e_test.go b/pkg/substrate/e2e_test.go index acd3b772..944c8a19 100644 --- a/pkg/substrate/e2e_test.go +++ b/pkg/substrate/e2e_test.go @@ -1,5 +1,5 @@ // The substrate client against a live cluster: files go in and come out of an environment intact, -// whatever their size, a process in it works on them, and the caller may list what a sweep needs. +// whatever their size, a process in it works on them, and the caller may list what a sweep and an inventory need. package substrate_test import ( @@ -98,3 +98,20 @@ func TestSweepListsTheActorsOfACluster(t *testing.T) { require.NoError(t, err) assert.Empty(t, swept) } + +// A caller's identity may list the cluster's workers and the actors of every atespace, as an +// inventory of the substrate must. +func TestInventoryOfACluster(t *testing.T) { + cluster := substratetest.FromEnv(t) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + client, err := substrate.Dial(cluster.Options) + require.NoError(t, err) + defer client.Close() + + workers, err := client.ListWorkers(ctx) + require.NoError(t, err) + assert.NotEmpty(t, workers, "a cluster that runs environments has workers") + _, err = client.ListActors(ctx, "") + require.NoError(t, err) +} diff --git a/pkg/substrate/inventory.go b/pkg/substrate/inventory.go new file mode 100644 index 00000000..6bebd983 --- /dev/null +++ b/pkg/substrate/inventory.go @@ -0,0 +1,124 @@ +// Lists what a substrate holds: the actors of one atespace or of every atespace, and the workers +// they run on, with each worker's capacity and how much of it its actors are allocated. +package substrate + +import ( + "context" + "fmt" + "strings" + "time" + + ateapipb "github.com/flanksource/captain/pkg/substrate/internal/ateapi" +) + +// Actor is one actor in a substrate: an environment, or anything else Substrate runs. +type Actor struct { + Atespace string `json:"atespace"` + Name string `json:"name"` + // State is resuming, running, suspending, suspended, crashed, deleting or reverting; + // unspecified when Substrate doesn't say, or the state's number for one this client doesn't know. + State string `json:"state"` + // Created is when the actor was created; zero when Substrate doesn't say. + Created time.Time `json:"created,omitzero"` +} + +// Worker is one of substrate's workers, the pods that actors run on. Name is Substrate's name for +// it. +type Worker struct { + Name string `json:"name"` + Pool string `json:"pool"` + Node string `json:"node"` + // State is active, draining or unavailable; unspecified when the worker hasn't reported one, + // or the state's number for one this client doesn't know. + State string `json:"state"` + // Capacity is what the worker can hold, and Allocated what its actors are allocated of it, + // their runtime's overhead included. + Capacity Resources `json:"capacity"` + Allocated Resources `json:"allocated"` +} + +// Resources is an amount of a worker's CPU, memory and actor slots. +type Resources struct { + CPUMillicores int64 `json:"cpuMillicores"` + MemoryBytes int64 `json:"memoryBytes"` + Actors int32 `json:"actors"` +} + +// listPageSize is how many items one list call asks Substrate for. +const listPageSize = 100 + +// ListActors lists the actors of atespace, or of every atespace when atespace is empty. +func (c *Client) ListActors(ctx context.Context, atespace string) ([]Actor, error) { + request := &ateapipb.ListActorsRequest{Atespace: atespace, Options: &ateapipb.ListOptions{PageSize: listPageSize}} + return list(func(token string) ([]Actor, string, error) { + request.Options.PageToken = token + page, err := c.control.ListActors(ctx, request) + if err != nil { + if atespace == "" { + return nil, "", fmt.Errorf("list the actors of every atespace: %w", err) + } + return nil, "", fmt.Errorf("list the actors of atespace %s: %w", atespace, err) + } + actors := make([]Actor, 0, len(page.GetActors())) + for _, actor := range page.GetActors() { + metadata := actor.GetMetadata() + listed := Actor{Atespace: metadata.GetAtespace(), Name: metadata.GetName(), State: stateName(actor.GetStatus().GetState().String(), "ACTOR_STATE_")} + if metadata.GetCreateTime() != nil { + listed.Created = metadata.GetCreateTime().AsTime() + } + actors = append(actors, listed) + } + return actors, page.GetNextPageToken(), nil + }) +} + +// ListWorkers lists the substrate's workers. +func (c *Client) ListWorkers(ctx context.Context) ([]Worker, error) { + request := &ateapipb.ListWorkersRequest{Options: &ateapipb.ListOptions{PageSize: listPageSize}} + return list(func(token string) ([]Worker, string, error) { + request.Options.PageToken = token + page, err := c.control.ListWorkers(ctx, request) + if err != nil { + return nil, "", fmt.Errorf("list substrate's workers: %w", err) + } + workers := make([]Worker, 0, len(page.GetWorkers())) + for _, worker := range page.GetWorkers() { + status := worker.GetStatus() + workers = append(workers, Worker{ + Name: worker.GetMetadata().GetName(), Pool: worker.GetWorkerPool(), Node: worker.GetNodeName(), + State: stateName(status.GetState().String(), "WORKER_STATE_"), + Capacity: resources(status.GetCapacity()), + Allocated: resources(status.GetAllocated()), + }) + } + return workers, page.GetNextPageToken(), nil + }) +} + +// list collects every page that page returns, starting from the empty token and following each +// page's next token until it is empty. +func list[T any](page func(token string) ([]T, string, error)) ([]T, error) { + var all []T + token := "" + for { + items, next, err := page(token) + if err != nil { + return nil, err + } + all = append(all, items...) + if next == "" { + return all, nil + } + token = next + } +} + +// stateName is the lower-case name of a Substrate enum value without its prefix, e.g. running +// for ACTOR_STATE_RUNNING. +func stateName(value, prefix string) string { + return strings.ToLower(strings.TrimPrefix(value, prefix)) +} + +func resources(r *ateapipb.WorkerResources) Resources { + return Resources{CPUMillicores: r.GetCpuMillicores(), MemoryBytes: r.GetMemoryBytes(), Actors: r.GetActors()} +} diff --git a/pkg/substrate/inventory_test.go b/pkg/substrate/inventory_test.go new file mode 100644 index 00000000..3f7e9950 --- /dev/null +++ b/pkg/substrate/inventory_test.go @@ -0,0 +1,89 @@ +// Specs for what a substrate holds, against an in-process Substrate: the actors of one atespace or +// of all of them, and the workers with their capacity and what actors use of it. +package substrate_test + +import ( + "encoding/json" + "testing" + "time" + + "github.com/flanksource/captain/pkg/substrate" + "github.com/flanksource/captain/pkg/substrate/substratetest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestListActorsListsOneAtespaceOrEveryAtespace(t *testing.T) { + server := substratetest.Start(t) + server.ListPageSize = 1 + ctx := testContext(t) + for atespace, ids := range map[string][]string{"ate-env": {"run-2", "run-1"}, "gavel": {"run-3"}} { + client := dial(t, server, func(o *substrate.Options) { o.Atespace = atespace }) + for _, id := range ids { + _, err := client.Create(ctx, substrate.Environment{ID: id}) + require.NoError(t, err) + } + } + client := dial(t, server) + names := func(actors []substrate.Actor) []string { + var names []string + for _, actor := range actors { + names = append(names, actor.Atespace+"/"+actor.Name) + } + return names + } + + inOne, err := client.ListActors(ctx, "ate-env") + require.NoError(t, err) + assert.Equal(t, []string{"ate-env/run-1", "ate-env/run-2"}, names(inOne)) + + all, err := client.ListActors(ctx, "") + require.NoError(t, err) + assert.Equal(t, []string{"ate-env/run-1", "ate-env/run-2", "gavel/run-3"}, names(all)) + assert.Equal(t, "running", all[0].State) + assert.WithinDuration(t, time.Now(), all[0].Created, time.Minute) +} + +func TestListWorkersReportsTheirCapacityAndWhatActorsUse(t *testing.T) { + server := substratetest.Start(t) + server.ListPageSize = 1 + workers := []substrate.Worker{ + { + Name: "pool-a-0", Pool: "pool-a", Node: "node-1", State: "active", + Capacity: substrate.Resources{CPUMillicores: 4000, MemoryBytes: 8 << 30, Actors: 10}, + Allocated: substrate.Resources{CPUMillicores: 1500, MemoryBytes: 2 << 30, Actors: 3}, + }, + { + Name: "pool-a-1", Pool: "pool-a", Node: "node-2", State: "draining", + Capacity: substrate.Resources{CPUMillicores: 4000, MemoryBytes: 8 << 30, Actors: 10}, + }, + } + for _, worker := range workers { + server.AddWorker(worker) + } + + got, err := dial(t, server).ListWorkers(testContext(t)) + require.NoError(t, err) + assert.Equal(t, workers, got) +} + +func TestInventoryEncodesAsJSON(t *testing.T) { + worker, err := json.Marshal(substrate.Worker{ + Name: "pool-a-0", Pool: "pool-a", Node: "node-1", State: "active", + Capacity: substrate.Resources{CPUMillicores: 4000, MemoryBytes: 8, Actors: 10}, + Allocated: substrate.Resources{CPUMillicores: 1500, MemoryBytes: 2, Actors: 3}, + }) + require.NoError(t, err) + assert.JSONEq(t, `{"name":"pool-a-0","pool":"pool-a","node":"node-1","state":"active", + "capacity":{"cpuMillicores":4000,"memoryBytes":8,"actors":10}, + "allocated":{"cpuMillicores":1500,"memoryBytes":2,"actors":3}}`, string(worker)) + + created := time.Date(2026, 11, 11, 10, 0, 0, 0, time.UTC) + actors, err := json.Marshal([]substrate.Actor{ + {Atespace: "ate-env", Name: "run-1", State: "running", Created: created}, + {Atespace: "ate-env", Name: "run-2", State: "crashed"}, + }) + require.NoError(t, err) + assert.JSONEq(t, `[{"atespace":"ate-env","name":"run-1","state":"running","created":"2026-11-11T10:00:00Z"}, + {"atespace":"ate-env","name":"run-2","state":"crashed"}]`, string(actors), "a zero creation time is left out") +} diff --git a/pkg/substrate/substrate.go b/pkg/substrate/substrate.go index 5229298d..471e4de4 100644 --- a/pkg/substrate/substrate.go +++ b/pkg/substrate/substrate.go @@ -177,10 +177,10 @@ func (c *Client) Create(ctx context.Context, environment Environment) (*Env, err func (e *Env) ID() string { return e.id } // Actor is the environment's SPIFFE ID, which the egress gateway names it by. -func (e *Env) Actor() string { return Actor(e.client.options.Atespace, e.id) } +func (e *Env) Actor() string { return SPIFFEID(e.client.options.Atespace, e.id) } -// Actor is the SPIFFE ID of the environment id in atespace. -func Actor(atespace, id string) string { +// SPIFFEID is the SPIFFE ID of the environment id in atespace. +func SPIFFEID(atespace, id string) string { return "spiffe://substrate-actor.local/actor/" + atespace + "/" + id } @@ -206,23 +206,15 @@ const fileChunk = 256 * 1024 // actor was created before cutoff: the ones a caller that died mid-run left behind, once no run // of theirs can still be live. It returns the IDs it deleted, and an error for each it couldn't. func (c *Client) Sweep(ctx context.Context, prefix string, cutoff time.Time) ([]string, error) { + actors, err := c.ListActors(ctx, c.options.Atespace) + if err != nil { + return nil, err + } var stale []string - request := &ateapipb.ListActorsRequest{Atespace: c.options.Atespace, Options: &ateapipb.ListOptions{PageSize: 100}} - for { - page, err := c.control.ListActors(ctx, request) - if err != nil { - return nil, fmt.Errorf("list the actors of atespace %s: %w", c.options.Atespace, err) - } - for _, actor := range page.GetActors() { - metadata := actor.GetMetadata() - if strings.HasPrefix(metadata.GetName(), prefix) && metadata.GetCreateTime() != nil && metadata.GetCreateTime().AsTime().Before(cutoff) { - stale = append(stale, metadata.GetName()) - } - } - if page.GetNextPageToken() == "" { - break + for _, actor := range actors { + if strings.HasPrefix(actor.Name, prefix) && !actor.Created.IsZero() && actor.Created.Before(cutoff) { + stale = append(stale, actor.Name) } - request.Options.PageToken = page.GetNextPageToken() } var swept []string var errs []error diff --git a/pkg/substrate/substratetest/server.go b/pkg/substrate/substratetest/server.go index 46b74303..37117018 100644 --- a/pkg/substrate/substratetest/server.go +++ b/pkg/substrate/substratetest/server.go @@ -3,6 +3,7 @@ package substratetest import ( + "cmp" "context" "crypto/ecdsa" "crypto/elliptic" @@ -49,7 +50,8 @@ type Server struct { // StreamLimit, when positive, ends every StreamProcessOutput stream after that many output // messages, as Substrate's router ends a long stream, so a client has to reconnect. StreamLimit int - // ListPageSize, when positive, caps every ListActors page, so a client has to page. + // ListPageSize, when positive, caps every ListActors and ListWorkers page, so a client has to + // page. ListPageSize int envAddr, controlAddr string @@ -59,6 +61,7 @@ type Server struct { envs map[string]*Environment policies map[string]*ateapipb.EgressPolicy deleted []string + workers []*ateapipb.Worker } // Environment is one environment the server created. @@ -124,6 +127,22 @@ func (s *Server) Policy(atespace, id string) (*ateapipb.EgressPolicy, bool) { return policy, ok } +// AddWorker adds worker to the workers the control plane lists. +func (s *Server) AddWorker(worker substrate.Worker) { + s.mu.Lock() + defer s.mu.Unlock() + resources := func(r substrate.Resources) *ateapipb.WorkerResources { + return &ateapipb.WorkerResources{CpuMillicores: r.CPUMillicores, MemoryBytes: r.MemoryBytes, Actors: r.Actors} + } + s.workers = append(s.workers, &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: worker.Name}, WorkerPool: worker.Pool, NodeName: worker.Node, + Status: &ateapipb.WorkerStatus{ + State: ateapipb.WorkerState(ateapipb.WorkerState_value["WORKER_STATE_"+strings.ToUpper(worker.State)]), + Capacity: resources(worker.Capacity), Allocated: resources(worker.Allocated), + }, + }) +} + // Deleted lists the environments deleted so far, as atespace/id. func (s *Server) Deleted() []string { s.mu.Lock() @@ -224,43 +243,71 @@ func (c control) CreateActorEgressPolicy(_ context.Context, req *ateapipb.Create return req.GetEgressPolicy(), nil } -// ListActors lists the actors of the atespace's environments, in name order, a page at a time: -// the page token is the index of the page's first actor. +// ListActors lists the actors of the atespace's environments, or of every environment when the +// request names no atespace, in atespace and name order, a page at a time. Every actor is running. func (c control) ListActors(_ context.Context, req *ateapipb.ListActorsRequest) (*ateapipb.ListActorsResponse, error) { s := c.s s.mu.Lock() defer s.mu.Unlock() var envs []*Environment for _, env := range s.envs { - if env.Atespace == req.GetAtespace() { + if req.GetAtespace() == "" || env.Atespace == req.GetAtespace() { envs = append(envs, env) } } - slices.SortFunc(envs, func(a, b *Environment) int { return strings.Compare(a.ID, b.ID) }) - start := 0 - if token := req.GetOptions().GetPageToken(); token != "" { - var err error - if start, err = strconv.Atoi(token); err != nil || start < 0 || start > len(envs) { - return nil, status.Errorf(codes.InvalidArgument, "page token %q", token) + slices.SortFunc(envs, func(a, b *Environment) int { + return cmp.Or(strings.Compare(a.Atespace, b.Atespace), strings.Compare(a.ID, b.ID)) + }) + start, end, next, err := s.page(req.GetOptions(), len(envs)) + if err != nil { + return nil, err + } + response := &ateapipb.ListActorsResponse{NextPageToken: next} + for _, env := range envs[start:end] { + response.Actors = append(response.Actors, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: env.Atespace, Name: env.ID, CreateTime: timestamppb.New(env.Created)}, + Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING}, + }) + } + return response, nil +} + +// ListWorkers lists the workers added with AddWorker, in the order they were added, a page at a +// time. +func (c control) ListWorkers(_ context.Context, req *ateapipb.ListWorkersRequest) (*ateapipb.ListWorkersResponse, error) { + s := c.s + s.mu.Lock() + defer s.mu.Unlock() + start, end, next, err := s.page(req.GetOptions(), len(s.workers)) + if err != nil { + return nil, err + } + response := &ateapipb.ListWorkersResponse{NextPageToken: next} + for _, worker := range s.workers[start:end] { + response.Workers = append(response.Workers, proto.Clone(worker).(*ateapipb.Worker)) + } + return response, nil +} + +// page is the range of a list of n items that options' page covers, and the token of the page +// after it: a page token is the index of the page's first item. +func (s *Server) page(options *ateapipb.ListOptions, n int) (start, end int, next string, err error) { + if token := options.GetPageToken(); token != "" { + if start, err = strconv.Atoi(token); err != nil || start < 0 || start > n { + return 0, 0, "", status.Errorf(codes.InvalidArgument, "page token %q", token) } } - end := len(envs) - if size := int(req.GetOptions().GetPageSize()); size > 0 && start+size < end { + end = n + if size := int(options.GetPageSize()); size > 0 && start+size < end { end = start + size } if s.ListPageSize > 0 && start+s.ListPageSize < end { end = start + s.ListPageSize } - response := &ateapipb.ListActorsResponse{} - for _, env := range envs[start:end] { - response.Actors = append(response.Actors, &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{ - Atespace: env.Atespace, Name: env.ID, CreateTime: timestamppb.New(env.Created), - }}) - } - if end < len(envs) { - response.NextPageToken = strconv.Itoa(end) + if end < n { + next = strconv.Itoa(end) } - return response, nil + return start, end, next, nil } // process is one process started in an environment, with its output spooled.