Skip to content
Closed
6 changes: 4 additions & 2 deletions pkg/budgets/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func normalizeInput(input RuleInput) (RuleInput, error) {
if input.Name == "" || input.Amount <= 0 || input.Window == "" {
return RuleInput{}, fmt.Errorf("%w: name, positive amount, and window are required", ErrInvalid)
}
if _, err := windowStart(input.Window, time.Now().UTC()); err != nil {
if _, err := WindowStart(input.Window, time.Now().UTC()); err != nil {
return RuleInput{}, fmt.Errorf("%w: %v", ErrInvalid, err)
}
dimensions := make(map[string]string, len(input.Match.Dimensions))
Expand Down Expand Up @@ -95,7 +95,9 @@ func cleanList(values []string) []string {
return out
}

func windowStart(raw string, now time.Time) (time.Time, error) {
// WindowStart is where a budget window that ends at now starts: raw is datemath relative to now,
// such as now/M or now-30d, evaluated in UTC. An absolute or future start is refused.
func WindowStart(raw string, now time.Time) (time.Time, error) {
raw = strings.TrimSpace(raw)
if !strings.HasPrefix(raw, "now") {
return time.Time{}, fmt.Errorf("budget window %q must be relative to now", raw)
Expand Down
37 changes: 37 additions & 0 deletions pkg/budgets/window_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Tests where a budget window starts: relative datemath evaluated in UTC, and why an absolute,
// unparseable or future window is refused.
package budgets_test

import (
"testing"
"time"

"github.com/flanksource/captain/pkg/budgets"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestWindowStart(t *testing.T) {
now := time.Date(2026, 11, 11, 15, 30, 0, 0, time.UTC)

for window, want := range map[string]time.Time{
"now/d": time.Date(2026, 11, 11, 0, 0, 0, 0, time.UTC),
"now/M": time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC),
"now-30d": now.AddDate(0, 0, -30),
" now/d ": time.Date(2026, 11, 11, 0, 0, 0, 0, time.UTC),
} {
start, err := budgets.WindowStart(window, now)
require.NoError(t, err, window)
assert.Equal(t, want, start, window)
}

for window, reason := range map[string]string{
"2026-11-01": `budget window "2026-11-01" must be relative to now`,
"now+1d": `budget window "now+1d" starts after now`,
"now-x": `invalid budget window "now-x": `,
} {
_, err := budgets.WindowStart(window, now)
require.Error(t, err, window)
assert.Contains(t, err.Error(), reason, window)
}
}
59 changes: 59 additions & 0 deletions pkg/database/budget_spend_integration_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Specs for a budget rule's spend from the model-call ledger: the calls attributed to one of its
// groups since a time, priced by ModelCallCostUSD, and refused when any of them isn't in USD.
package database

import (
"context"
"time"

"github.com/flanksource/captain/pkg/api"
"github.com/google/uuid"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var _ = Describe("Budget spend", func() {
const schema = "agent_budget_spend_context"
var (
db *DB
rule *BudgetRule
)

BeforeEach(func(ctx SpecContext) {
db = openSchemaScopedDatabases(ctx, "captain_budget_spend", schema)[0]
var err error
rule, err = db.CreateBudgetRule(ctx, BudgetRuleInput{Name: "per org", GroupBy: []string{"org"}, Amount: 10, Window: "now-30d"})
Expect(err).NotTo(HaveOccurred())
})

attribute := func(ctx context.Context, callID uuid.UUID, org string) {
var turnID string
Expect(db.Gorm().Table(schema+".captain_model_calls").Where("id = ?", callID).Select("turn_id").Scan(&turnID).Error).To(Succeed())
Expect(db.SetModelCallBudgets(ctx, uuid.MustParse(turnID), callID, map[string]string{"org": org},
[]BudgetAttribution{{RuleID: rule.ID, GroupValues: map[string]string{"org": org}}})).To(Succeed())
}

It("totals the calls attributed to a group since a time", func(ctx SpecContext) {
since := time.Now().Add(-time.Hour)
attribute(ctx, recordModelCall(ctx, db, "acme provider priced", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 1}, 0.04), "acme")
attribute(ctx, recordModelCall(ctx, db, "acme list priced", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 0.01, OutputCost: 0.02}, 0), "acme")
attribute(ctx, recordModelCall(ctx, db, "globex", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 5}, 0), "globex")

spend, err := db.BudgetSpendUSD(ctx, rule.ID, map[string]string{"org": "acme"}, since)
Expect(err).NotTo(HaveOccurred())
Expect(spend).To(BeNumerically("~", 0.07, 0.000000001))

later, err := db.BudgetSpendUSD(ctx, rule.ID, map[string]string{"org": "acme"}, time.Now().Add(time.Hour))
Expect(err).NotTo(HaveOccurred())
Expect(later).To(BeZero())
})

It("refuses a group holding a call outside USD", func(ctx SpecContext) {
inEuros := recordModelCall(ctx, db, "acme in euros", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 5}, 0)
Expect(db.Gorm().Table(schema+".captain_model_calls").Where("id = ?", inEuros).Update("currency", "EUR").Error).To(Succeed())
attribute(ctx, inEuros, "acme")

_, err := db.BudgetSpendUSD(ctx, rule.ID, map[string]string{"org": "acme"}, time.Now().Add(-time.Hour))
Expect(err).To(MatchError("budget ledger contains 1 completed non-USD model calls"))
})
})
23 changes: 11 additions & 12 deletions pkg/database/budget_store.go
Original file line number Diff line number Diff line change
Expand Up @@ -259,25 +259,24 @@ func (db *DB) budgetSpendStatement(ctx context.Context, ruleID uuid.UUID, groupJ

// BudgetSpendUSD derives completed spend from the model-call ledger. Any
// non-USD completed call in the group fails closed because no conversion rate
// is authoritative here.
// is authoritative here. One statement counts those calls and totals the spend,
// so a call completing between the two can't be priced at zero unnoticed.
func (db *DB) BudgetSpendUSD(ctx context.Context, ruleID uuid.UUID, groupValues map[string]string, since time.Time) (float64, error) {
groupJSON, err := json.Marshal(cloneStrings(groupValues))
if err != nil {
return 0, fmt.Errorf("encode budget group: %w", err)
}
var nonUSD int64
if err := db.budgetSpendStatement(ctx, ruleID, groupJSON, since).
Where("upper(calls.currency) <> 'USD'").Count(&nonUSD).Error; err != nil {
return 0, fmt.Errorf("inspect budget ledger currency: %w", err)
}
if nonUSD > 0 {
return 0, fmt.Errorf("budget ledger contains %d completed non-USD model calls", nonUSD)
var spend struct {
NonUSD int64
Total float64
}
var total float64
if err := db.budgetSpendStatement(ctx, ruleID, groupJSON, since).
Select(`COALESCE(SUM(CASE WHEN calls.provider_cost_usd > 0 THEN calls.provider_cost_usd ELSE calls.input_cost + calls.output_cost + calls.reasoning_cost + calls.cache_read_cost + calls.cache_write_cost END), 0)`).
Scan(&total).Error; err != nil {
Select("count(*) FILTER (WHERE upper(calls.currency) <> 'USD') AS non_usd, COALESCE(SUM(" + ModelCallCostUSD + "), 0) AS total").
Scan(&spend).Error; err != nil {
return 0, fmt.Errorf("aggregate completed budget spend: %w", err)
}
return total, nil
if spend.NonUSD > 0 {
return 0, fmt.Errorf("budget ledger contains %d completed non-USD model calls", spend.NonUSD)
}
return spend.Total, nil
}
21 changes: 12 additions & 9 deletions pkg/database/model_usage.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,17 +32,20 @@ type ModelUsageQuery struct {
Schemas []string
}

// modelUsageTotalsSelect mirrors api.Cost.Total() and the captain_session_costs
// view: prefer the provider-reported cost per call, with the list-price bucket
// sum as the fallback. Decided per row, not per group, so a mix of list-priced
// and provider-reported calls still totals. Non-USD calls contribute tokens but
// no cost — there is no rate table to convert them, and adding EUR to USD would
// be worse than reporting less.
const modelUsageTotalsSelect = `COALESCE(SUM(calls.input_tokens + calls.output_tokens + calls.reasoning_tokens + calls.cache_read_tokens + calls.cache_write_tokens), 0) AS total_tokens,
COALESCE(SUM(CASE WHEN upper(calls.currency) = 'USD' THEN
// ModelCallCostUSD is SQL for the USD cost of one row of captain_model_calls aliased calls. It
// mirrors api.Cost.Total() and the captain_session_costs view: the provider-reported cost when
// there is one, else the sum of the list-price buckets. Decided per row, not per group, so a mix
// of list-priced and provider-reported calls still totals. Non-USD calls cost nothing here: there
// is no rate table to convert them, and adding EUR to USD would be worse than reporting less.
const ModelCallCostUSD = `CASE WHEN upper(calls.currency) = 'USD' THEN
CASE WHEN calls.provider_cost_usd > 0 THEN calls.provider_cost_usd
ELSE calls.input_cost + calls.output_cost + calls.reasoning_cost + calls.cache_read_cost + calls.cache_write_cost END
ELSE 0 END), 0) AS total_cost_usd`
ELSE 0 END`

// modelUsageTotalsSelect totals the selected calls' tokens and their ModelCallCostUSD. Non-USD
// calls contribute tokens but no cost.
const modelUsageTotalsSelect = `COALESCE(SUM(calls.input_tokens + calls.output_tokens + calls.reasoning_tokens + calls.cache_read_tokens + calls.cache_write_tokens), 0) AS total_tokens,
COALESCE(SUM(` + ModelCallCostUSD + `), 0) AS total_cost_usd`

// ModelUsage aggregates Captain model calls in the selected schemas.
// Configured schemas that have never been migrated have no usage and are
Expand Down
72 changes: 47 additions & 25 deletions pkg/database/model_usage_integration_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package database

import (
"context"
"time"

"github.com/flanksource/captain/pkg/api"
Expand All @@ -9,6 +10,31 @@ import (
. "github.com/onsi/gomega"
)

// recordModelCall records one finished model call in a session titled title, with usage, its
// list-price cost, and the cost its provider reported, and returns the call's ID.
func recordModelCall(ctx context.Context, db *DB, title string, usage api.Usage, cost *api.Cost, providerCostUSD float64) uuid.UUID {
session, err := db.CreateOrGetSession(ctx, CreateSessionInput{
ProviderSessionID: title, Source: "codex", Provider: "openai",
HostID: "usage-scope-test", Title: title,
})
Expect(err).NotTo(HaveOccurred())
turn, _, err := db.CreateChatTurn(ctx, CreateChatTurnInput{
SessionID: session.ID, ProviderTurnID: title + " turn",
})
Expect(err).NotTo(HaveOccurred())
run, err := db.CreatePromptRun(ctx, CreatePromptRunInput{SessionID: session.ID, TurnID: &turn.ID})
Expect(err).NotTo(HaveOccurred())
callID, err := db.CreateChatModelCall(ctx, CreateChatModelCallInput{
TurnID: turn.ID, PromptRunID: run.ID, Model: "model-usage-scope", Provider: "openai", Mode: "cli",
})
Expect(err).NotTo(HaveOccurred())
Expect(db.FinishChatModelCall(ctx, FinishChatModelCallInput{
ID: callID, Status: ModelCallStatusSucceeded, StopReason: "end_turn",
Event: api.Event{Usage: &usage, CostUSD: providerCostUSD}, Cost: cost,
})).To(Succeed())
return callID
}

var _ = Describe("Model usage aggregation", func() {
It("counts only the sessions an owner's subquery selects", func(ctx SpecContext) {
const (
Expand All @@ -25,31 +51,8 @@ var _ = Describe("Model usage aggregation", func() {
db := openSchemaScopedDatabases(ctx, "captain_model_usage", "agent_usage_scope_context")[0]
since := time.Now().Add(-time.Hour)

record := func(title string, usage api.Usage, providerCostUSD float64) uuid.UUID {
session, err := db.CreateOrGetSession(ctx, CreateSessionInput{
ProviderSessionID: title, Source: "codex", Provider: "openai",
HostID: "usage-scope-test", Title: title,
})
Expect(err).NotTo(HaveOccurred())
turn, _, err := db.CreateChatTurn(ctx, CreateChatTurnInput{
SessionID: session.ID, ProviderTurnID: title + " turn",
})
Expect(err).NotTo(HaveOccurred())
run, err := db.CreatePromptRun(ctx, CreatePromptRunInput{SessionID: session.ID, TurnID: &turn.ID})
Expect(err).NotTo(HaveOccurred())
callID, err := db.CreateChatModelCall(ctx, CreateChatModelCallInput{
TurnID: turn.ID, PromptRunID: run.ID, Model: "model-usage-scope", Provider: "openai", Mode: "cli",
})
Expect(err).NotTo(HaveOccurred())
Expect(db.FinishChatModelCall(ctx, FinishChatModelCallInput{
ID: callID, Status: ModelCallStatusSucceeded, StopReason: "end_turn",
Event: api.Event{Usage: &usage, CostUSD: providerCostUSD}, Cost: &api.Cost{},
})).To(Succeed())
return session.ID
}

record(tenantTitle, api.Usage{InputTokens: tenantInputTokens, OutputTokens: tenantOutputToken}, tenantProviderCostUSD)
record(otherTenantTitle, api.Usage{InputTokens: 500, OutputTokens: 500}, otherProviderCostUSD)
recordModelCall(ctx, db, tenantTitle, api.Usage{InputTokens: tenantInputTokens, OutputTokens: tenantOutputToken}, &api.Cost{}, tenantProviderCostUSD)
recordModelCall(ctx, db, otherTenantTitle, api.Usage{InputTokens: 500, OutputTokens: 500}, &api.Cost{}, otherProviderCostUSD)

// Stands in for an embedding application's ownership table: it knows
// which sessions belong to the tenant, Captain knows what they cost.
Expand All @@ -70,4 +73,23 @@ var _ = Describe("Model usage aggregation", func() {
Expect(unscoped.TotalTokens).To(Equal(tenantInputTokens + tenantOutputToken + 1000))
Expect(unscoped.TotalCostUSD).To(BeNumerically("~", tenantProviderCostUSD+otherProviderCostUSD, 0.000000001))
})

It("prices each call by ModelCallCostUSD: its provider's cost, else its list price, and nothing outside USD", func(ctx SpecContext) {
const schema = "agent_call_cost_context"
db := openSchemaScopedDatabases(ctx, "captain_model_call_cost", schema)[0]
providerPriced := recordModelCall(ctx, db, "provider priced", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 1}, 0.04)
listPriced := recordModelCall(ctx, db, "list priced", api.Usage{InputTokens: 10, OutputTokens: 5}, &api.Cost{InputCost: 0.01, OutputCost: 0.02}, 0)
inEuros := recordModelCall(ctx, db, "in euros", api.Usage{InputTokens: 10}, &api.Cost{InputCost: 5}, 0)
Expect(db.Gorm().Table(schema+".captain_model_calls").Where("id = ?", inEuros).Update("currency", "EUR").Error).To(Succeed())

cost := func(id uuid.UUID) float64 {
var usd float64
Expect(db.Gorm().Table(schema+".captain_model_calls AS calls").Where("calls.id = ?", id).
Select(ModelCallCostUSD).Scan(&usd).Error).To(Succeed())
return usd
}
Expect(cost(providerPriced)).To(BeNumerically("~", 0.04, 0.000000001))
Expect(cost(listPriced)).To(BeNumerically("~", 0.03, 0.000000001))
Expect(cost(inEuros)).To(BeZero())
})
})
5 changes: 5 additions & 0 deletions pkg/substrate/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,11 @@ An `Env` moves files in and out (`WriteFile`, `ReadFile`, both streamed) and sta
stdin of a process started with `Stdin` (`Input`, one call per write, so an ended stream loses
nothing). It waits for its exit code (`Wait`) and kills its process group (`Kill`). `Delete` deletes the environment.

`Client.ListActors(atespace)` lists the actors of an atespace, or of every atespace when it is
empty, with their state and create time. `Client.ListWorkers()` lists the workers they run on, with
each one's pool, node, state, and capacity and allocation (CPU, memory and actor slots). `Sweep`
deletes the environments a dead caller left behind, from the atespace's actor list.

## Generated API

`internal/ateapi`, `internal/credprovider` and `internal/ateenvv1alpha` are generated by `go
Expand Down
19 changes: 18 additions & 1 deletion pkg/substrate/e2e_test.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// The substrate client against a live cluster: files go in and come out of an environment intact,
// whatever their size, a process in it works on them, and the caller may list what a sweep needs.
// whatever their size, a process in it works on them, and the caller may list what a sweep and an inventory need.
package substrate_test

import (
Expand Down Expand Up @@ -98,3 +98,20 @@ func TestSweepListsTheActorsOfACluster(t *testing.T) {
require.NoError(t, err)
assert.Empty(t, swept)
}

// A caller's identity may list the cluster's workers and the actors of every atespace, as an
// inventory of the substrate must.
func TestInventoryOfACluster(t *testing.T) {
cluster := substratetest.FromEnv(t)
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
client, err := substrate.Dial(cluster.Options)
require.NoError(t, err)
defer client.Close()

workers, err := client.ListWorkers(ctx)
require.NoError(t, err)
assert.NotEmpty(t, workers, "a cluster that runs environments has workers")
_, err = client.ListActors(ctx, "")
require.NoError(t, err)
}
Loading
Loading