diff --git a/.github/workflows/northbrook.yml b/.github/workflows/northbrook.yml new file mode 100644 index 00000000000..a0a865061dc --- /dev/null +++ b/.github/workflows/northbrook.yml @@ -0,0 +1,26 @@ +name: Northbrook tests and build + +on: + push: + branches: [northbrook] + pull_request: + branches: [northbrook] + workflow_dispatch: + +permissions: + contents: read + +jobs: + test-build: + if: github.repository == 'north-brook/CLIProxyAPI' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - name: Test + run: go test ./... + - name: Build + run: go build -o test-output ./cmd/server && rm test-output diff --git a/NORTHBROOK.md b/NORTHBROOK.md new file mode 100644 index 00000000000..19f155454ea --- /dev/null +++ b/NORTHBROOK.md @@ -0,0 +1,87 @@ +# Northbrook quota-aware routing + +This fork adds a standalone Claude OAuth routing strategy on top of upstream +CLIProxyAPI v8.0.43. Codex WebSockets and the other existing transports are preserved. + +```yaml +routing: + strategy: quota-aware + session-affinity: true + session-affinity-ttl: 24h + session-affinity-subagents: true + quota-aware: + refresh-interval: 5m + max-age: 15m + reserves: + - provider: claude + account: personal@example.com + percent: 20 +``` + +Account matching uses the OAuth credential's email, case insensitively. No +credentials or personal account configuration belong in this repository. + +## Selection + +For each eligible Claude OAuth account, each applicable quota window contributes: + +``` +spend rate = (remaining percentage - reserved percentage) / seconds until reset +account score = minimum spend rate across its applicable windows +``` + +The highest score wins a new binding. This favors allowance that can be spent +faster before it expires, while respecting the most restrictive quota window. +Five-hour and shared weekly windows are required for a protected account; +reported Opus/Sonnet weekly windows also apply when routing that model family. +An unused zero-utilization window without a reset is treated as a full-duration +window until the provider reports its actual first-request reset. + +Fixed account priorities only break equal scores or provide a fallback when no +unprotected account has fresh, complete quota data. Other providers and Claude +API keys retain priority-based round-robin behavior. A healthy session keeps its +existing account even when another account's score rises. + +## Reserves and freshness + +A request cannot select an account at or below its reserve in any applicable +window. This check precedes explicit and inferred session affinity, subagent +inheritance, pinned account selection, and retries. When necessary, a session +moves to another eligible account. If none is eligible, the proxy returns a +`quota_reserve` error rather than consuming reserved allowance. + +Protected accounts with missing, invalid, expired, or stale quota data are excluded. +Unprotected accounts with unknown quota remain available as a fallback. Existing +provider cooldowns, model availability, and disabled-account rules still apply. +Reserves require the quota-aware strategy; invalid configuration is rejected on +startup and reload. Home dispatch is not supported because it owns account +selection outside this process. + +Quota headers from ordinary Claude responses update the existing passive quota +snapshot. A background task also reads the authenticated Claude usage endpoint +every five minutes by default, covering accounts used outside the proxy. HTTP 429 +probes back off for at least fifteen minutes and honor a longer Retry-After. +There are no usage probes on the inference request path. Probe results cannot +overwrite a newer response snapshot or a replaced credential. + +The reserve is an admission threshold based on provider observations, not a +guarantee of an exact remaining balance: already-running requests, delayed +provider accounting, and simultaneous Claude app usage can cross it. Set a larger +reserve if additional margin is needed. The proxy does not cancel running work. + +Configuration changes use the existing file watcher or v8 management config API. +No management UI changes are required. The existing auth-file quota observations +expose the signals used by the router. + +## Build and verify + +Use Go 1.26.10 or newer and the repository's normal recipes: + +```sh +go test ./... +go test -race ./sdk/cliproxy/auth ./sdk/cliproxy -run 'TestQuota|TestClaudeUsageHeaders' +go build -o cli-proxy-api ./cmd/server +``` + +On macOS, point DEVELOPER_DIR and SDKROOT at a matching installed Xcode toolchain +if the default compiler cannot read the selected SDK's text-based libraries. diff --git a/config.example.yaml b/config.example.yaml index 7564a5eb085..03f32085e3c 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -147,7 +147,17 @@ access: # Routing strategy for selecting credentials when multiple match. routing: - strategy: "round-robin" # round-robin (default), weighted-round-robin, fill-first + strategy: "round-robin" # round-robin (default), weighted-round-robin, fill-first, quota-aware + # Northbrook: quota-aware ranks Claude OAuth accounts using spendable quota + # and time until reset. Reserves override session affinity and fixed priority. + # Other providers retain priority-based round-robin. Standalone mode only. + # quota-aware: + # refresh-interval: "5m" # background idle-account usage probes + # max-age: "15m" # protected accounts require fresh quota data + # reserves: + # - provider: "claude" + # account: "personal@example.com" # exact OAuth email, case insensitive + # percent: 20 # retain in every applicable window # weighted-round-robin uses each credential's integer weight (default 1, maximum 1,000,000). # Non-positive weights exclude the credential while this strategy is active. # For OAuth/file credentials, add a top-level numeric "weight" field to the auth JSON. diff --git a/internal/config/config_load.go b/internal/config/config_load.go index baca01e92e6..b67686e3c92 100644 --- a/internal/config/config_load.go +++ b/internal/config/config_load.go @@ -236,6 +236,9 @@ func LoadConfigOptional(configFile string, optional bool) (*Config, error) { if errValidate := cfg.Codex.LiveMediaRelay.Validate(); errValidate != nil { return nil, errValidate } + if errValidate := cfg.ValidateQuotaRouting(); errValidate != nil { + return nil, errValidate + } if errValidate := cfg.ValidateCredentialWeights(); errValidate != nil { return nil, errValidate } diff --git a/internal/config/config_types.go b/internal/config/config_types.go index fbd849f01cb..9ca294a2898 100644 --- a/internal/config/config_types.go +++ b/internal/config/config_types.go @@ -416,9 +416,12 @@ type QuotaExceeded struct { // RoutingConfig configures how credentials are selected for requests. type RoutingConfig struct { // Strategy selects the credential selection strategy. - // Supported values: "round-robin" (default), "weighted-round-robin", "fill-first". + // Supported values: "round-robin" (default), "weighted-round-robin", "fill-first", "quota-aware". Strategy string `yaml:"strategy,omitempty" json:"strategy,omitempty"` + // QuotaAware controls Claude OAuth quota routing and per-account reserves. + QuotaAware QuotaAwareRoutingConfig `yaml:"quota-aware,omitempty" json:"quota-aware,omitempty"` + // SessionAffinity enables universal session-sticky routing for all clients. // Explicit Claude Code, Codex, OpenCode, and pi session headers are preferred, // followed by prompt_cache_key, Responses conversation IDs, legacy body IDs, diff --git a/internal/config/parse.go b/internal/config/parse.go index c3442be1dd5..72994b52ac6 100644 --- a/internal/config/parse.go +++ b/internal/config/parse.go @@ -52,6 +52,9 @@ func ParseConfigBytes(data []byte) (*Config, error) { if errValidate := cfg.CredentialInFlight.Validate(); errValidate != nil { return nil, errValidate } + if errValidate := cfg.ValidateQuotaRouting(); errValidate != nil { + return nil, errValidate + } if errValidate := cfg.ValidateCredentialWeights(); errValidate != nil { return nil, errValidate } diff --git a/internal/config/quota_routing.go b/internal/config/quota_routing.go new file mode 100644 index 00000000000..f54c26d98c8 --- /dev/null +++ b/internal/config/quota_routing.go @@ -0,0 +1,79 @@ +package config + +import ( + "fmt" + "math" + "strings" + "time" +) + +// QuotaAwareRoutingConfig applies only when routing.strategy is quota-aware. +type QuotaAwareRoutingConfig struct { + RefreshInterval string `yaml:"refresh-interval,omitempty" json:"refresh-interval,omitempty"` + MaxAge string `yaml:"max-age,omitempty" json:"max-age,omitempty"` + Reserves []QuotaReserve `yaml:"reserves,omitempty" json:"reserves,omitempty"` +} + +// QuotaReserve identifies an OAuth account by provider and email, not filename. +type QuotaReserve struct { + Provider string `yaml:"provider" json:"provider"` + Account string `yaml:"account" json:"account"` + Percent float64 `yaml:"percent" json:"percent"` +} + +func (q QuotaAwareRoutingConfig) RefreshDuration() time.Duration { + if d, err := time.ParseDuration(q.RefreshInterval); err == nil && d > 0 { + return d + } + return 5 * time.Minute +} + +func (q QuotaAwareRoutingConfig) MaxAgeDuration() time.Duration { + if d, err := time.ParseDuration(q.MaxAge); err == nil && d > 0 { + return d + } + return 15 * time.Minute +} + +// ValidateQuotaRouting rejects typos and invalid reserves rather than silently +// disabling account protection. Other providers retain their existing routing. +func (cfg *Config) ValidateQuotaRouting() error { + if cfg == nil { + return nil + } + q := cfg.Routing.QuotaAware + quotaAware := strings.EqualFold(strings.TrimSpace(cfg.Routing.Strategy), "quota-aware") + if len(q.Reserves) > 0 && !quotaAware { + return fmt.Errorf("quota reserves require routing.strategy: quota-aware") + } + if quotaAware && cfg.Home.Enabled { + return fmt.Errorf("quota-aware routing is supported only in standalone mode, not Home dispatch") + } + for name, value := range map[string]string{"refresh-interval": q.RefreshInterval, "max-age": q.MaxAge} { + if value == "" { + continue + } + d, err := time.ParseDuration(value) + if err != nil || d < time.Second { + return fmt.Errorf("routing.quota-aware.%s must be a duration of at least 1s", name) + } + } + if q.MaxAgeDuration() < q.RefreshDuration() { + return fmt.Errorf("routing.quota-aware.max-age must be at least refresh-interval") + } + seen := map[string]bool{} + for _, r := range q.Reserves { + if r.Provider != "claude" || strings.TrimSpace(r.Account) == "" { + return fmt.Errorf("quota reserves require provider claude and a nonempty account email") + } + if math.IsNaN(r.Percent) || math.IsInf(r.Percent, 0) || r.Percent < 0 || r.Percent > 100 { + return fmt.Errorf("quota reserve percent must be between 0 and 100") + } + key := strings.ToLower(strings.TrimSpace(r.Account)) + if seen[key] { + return fmt.Errorf("duplicate quota reserve for account %q", r.Account) + } + seen[key] = true + } + return nil +} diff --git a/internal/config/quota_routing_test.go b/internal/config/quota_routing_test.go new file mode 100644 index 00000000000..4dde8d40a8d --- /dev/null +++ b/internal/config/quota_routing_test.go @@ -0,0 +1,50 @@ +package config + +import ( + "math" + "testing" + "time" + + "gopkg.in/yaml.v3" +) + +func TestQuotaRoutingValidationAndRoundTrip(t *testing.T) { + good := &Config{Routing: RoutingConfig{Strategy: "quota-aware", QuotaAware: QuotaAwareRoutingConfig{Reserves: []QuotaReserve{{Provider: "claude", Account: "personal@example.com", Percent: 20}}}}} + if err := good.ValidateQuotaRouting(); err != nil { + t.Fatal(err) + } + if good.Routing.QuotaAware.RefreshDuration() != 5*time.Minute || good.Routing.QuotaAware.MaxAgeDuration() != 15*time.Minute { + t.Fatal("unexpected quota observation defaults") + } + raw, err := yaml.Marshal(good) + if err != nil { + t.Fatal(err) + } + var restored Config + if err = yaml.Unmarshal(raw, &restored); err != nil || restored.Routing.QuotaAware.Reserves[0].Percent != 20 { + t.Fatalf("reserve lost on config roundtrip: %v", err) + } + for _, change := range []func(*Config){ + func(c *Config) { c.Routing.Strategy = "round-robin" }, + func(c *Config) { c.Home.Enabled = true }, + func(c *Config) { c.Routing.QuotaAware.Reserves[0].Percent = -1 }, + func(c *Config) { c.Routing.QuotaAware.Reserves[0].Percent = 101 }, + func(c *Config) { c.Routing.QuotaAware.Reserves[0].Percent = math.NaN() }, + func(c *Config) { c.Routing.QuotaAware.Reserves[0].Provider = "codex" }, + func(c *Config) { c.Routing.QuotaAware.Reserves[0].Account = "" }, + func(c *Config) { c.Routing.QuotaAware.RefreshInterval = "invalid" }, + func(c *Config) { c.Routing.QuotaAware.MaxAge = "1m" }, + func(c *Config) { + c.Routing.QuotaAware.Reserves = append(c.Routing.QuotaAware.Reserves, c.Routing.QuotaAware.Reserves[0]) + }, + } { + bad := good.CloneForRuntime() + change(bad) + if bad.ValidateQuotaRouting() == nil { + t.Fatalf("invalid quota config accepted: %+v", bad.Routing.QuotaAware) + } + } + if good.Routing.QuotaAware.Reserves[0].Percent != 20 { + t.Fatal("runtime clone shared reserve configuration") + } +} diff --git a/sdk/api/handlers/openai/openai_responses_interrupt_test.go b/sdk/api/handlers/openai/openai_responses_interrupt_test.go index f53fd269013..7cbfa6960a2 100644 --- a/sdk/api/handlers/openai/openai_responses_interrupt_test.go +++ b/sdk/api/handlers/openai/openai_responses_interrupt_test.go @@ -271,7 +271,7 @@ func (e *homeInterruptExecutor) InterruptExecutionSession(ctx context.Context, s type blockingHTTPInterruptExecutor struct { calls atomic.Int32 - canceled atomic.Bool + canceled chan struct{} } func (*blockingHTTPInterruptExecutor) Identifier() string { return "codex" } @@ -292,7 +292,7 @@ func (e *blockingHTTPInterruptExecutor) ExecuteStream(ctx context.Context, _ *co } chunks <- coreexecutor.StreamChunk{Payload: []byte(`{"type":"response.created","response":{"id":"r-http"}}`)} <-ctx.Done() - e.canceled.Store(true) + close(e.canceled) }() return &coreexecutor.StreamResult{Chunks: chunks}, nil } @@ -317,7 +317,7 @@ func (*blockingHTTPInterruptExecutor) InterruptExecutionSession(context.Context, // cancels an HTTP upstream turn and lets the same socket continue. func TestResponsesInterruptStopsHTTPUpstream(t *testing.T) { gin.SetMode(gin.TestMode) - executor := &blockingHTTPInterruptExecutor{} + executor := &blockingHTTPInterruptExecutor{canceled: make(chan struct{})} manager := coreauth.NewManager(nil, nil, nil) manager.SetConfig(&config.Config{}) manager.RegisterExecutor(executor) @@ -372,7 +372,9 @@ func TestResponsesInterruptStopsHTTPUpstream(t *testing.T) { if got := gjson.GetBytes(interrupted, "response.incomplete_details.reason").String(); got != "interrupted" { t.Fatalf("interrupt reason = %q, payload %s", got, interrupted) } - if !executor.canceled.Load() { + select { + case <-executor.canceled: + case <-time.After(5 * time.Second): t.Fatal("http upstream was not canceled") } if errSend := client.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf(`{"type":"response.create","model":%q,"previous_response_id":"r-http","input":[]}`, model))); errSend != nil { diff --git a/sdk/cliproxy/auth/conductor_selection.go b/sdk/cliproxy/auth/conductor_selection.go index 7cdacb7185f..33586482f82 100644 --- a/sdk/cliproxy/auth/conductor_selection.go +++ b/sdk/cliproxy/auth/conductor_selection.go @@ -628,6 +628,14 @@ func (m *Manager) availableAuthsForRouteModelWithPriorityMode(auths []*Auth, pro // unless session affinity or an across-priorities scheduler is active, in which case the selector // or scheduler additionally receives lower priority tiers. func (m *Manager) availableAuthsForSelector(selector Selector, auths []*Auth, provider, routeModel string, now time.Time, schedulerAcross bool) (priorityAuths, selectorAuths []*Auth, err error) { + if quota := quotaSelector(selector); quota != nil { + all, errAvailable := m.availableAuthsForRouteModelAcrossPriorities(auths, provider, routeModel, now) + if errAvailable != nil { + return nil, nil, errAvailable + } + eligible, errQuota := quota.eligible(cloneAuthSlice(all), routeModel, now) + return eligible, eligible, errQuota + } _, sessionAffinity := selector.(*SessionAffinitySelector) if !sessionAffinity && !schedulerAcross { @@ -694,6 +702,9 @@ func (m *Manager) transientlyUnavailableSessionAffinityAuthIDs(selector Selector if candidate == nil || candidate.ID == "" { continue } + if quota := quotaSelector(selector); quota != nil && quota.budget(candidate, routeModel, now).blocked { + continue + } checkModel := m.selectionModelForAuth(candidate, routeModel) if !isAuthBlockedByQuotaOrAuthForModel(candidate, checkModel, now) { if authIDs == nil { @@ -1023,6 +1034,11 @@ func (m *Manager) pickViaPluginScheduler(ctx context.Context, scheduler PluginSc if !okStrategy { return nil, false, nil } + // A built-in delegate selects from the scheduler's full pool. Keep quota + // routing on the filtered selector path so a delegate cannot spend reserves. + if quotaSelector(m.Selector()) != nil { + return nil, false, nil + } return m.pickViaBuiltinScheduler(ctx, strategy, providerKey, providers, model, opts, tried) } diff --git a/sdk/cliproxy/auth/selector.go b/sdk/cliproxy/auth/selector.go index fc7805413a6..1bc5b0937ac 100644 --- a/sdk/cliproxy/auth/selector.go +++ b/sdk/cliproxy/auth/selector.go @@ -1098,6 +1098,13 @@ func (s *SessionAffinitySelector) Trees() *cliproxysession.InMemorySessionTreeSt // a session uses multiple models (e.g., gemini-2.5-pro and gemini-3-flash-preview) // that may be supported by different auth credentials, and to avoid cross-provider conflicts. func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth) (*Auth, error) { + if quota := quotaSelector(s); quota != nil { + var errQuota error + auths, errQuota = quota.eligible(auths, model, time.Now()) + if errQuota != nil { + return nil, errQuota + } + } entry := selectorLogEntry(ctx) if opts.Metadata == nil { opts.Metadata = make(map[string]any) @@ -1146,7 +1153,8 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri availabilityCandidates = positiveWeightAuths(auths) } if primaryID == "" { - fallbackAuths, errAvailable := getSelectorAvailableAuths(ctx, availabilityCandidates, provider, model, now) + fallbackAuths, errAvailable := getSelectorAvailableAuthsAcrossPriorities(ctx, availabilityCandidates, provider, model, now) + fallbackAuths = routingFallbackCandidates(s.fallback, fallbackAuths) if errAvailable != nil { return nil, errAvailable } @@ -1160,7 +1168,7 @@ func (s *SessionAffinitySelector) Pick(ctx context.Context, provider, model stri if err != nil { return nil, err } - fallbackAuths := highestPriorityAuths(available) + fallbackAuths := routingFallbackCandidates(s.fallback, available) modelKey := canonicalModelKey(model) cacheKey := provider + "::" + primaryID + "::" + modelKey @@ -1371,7 +1379,7 @@ func (s *SessionAffinitySelector) pickLCP(ctx context.Context, provider, model s } } - fallbackAuths := highestPriorityAuths(available) + fallbackAuths := routingFallbackCandidates(s.fallback, available) auth, errPick := s.fallback.Pick(ctx, provider, model, opts, fallbackAuths) if errPick != nil { return nil, true, errPick diff --git a/sdk/cliproxy/auth/selector_quota.go b/sdk/cliproxy/auth/selector_quota.go new file mode 100644 index 00000000000..77d5aa93c9e --- /dev/null +++ b/sdk/cliproxy/auth/selector_quota.go @@ -0,0 +1,159 @@ +package auth + +import ( + "context" + "math" + "net/http" + "strconv" + "strings" + "time" + + "github.com/router-for-me/CLIProxyAPI/v8/internal/config" + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" +) + +// QuotaAwareSelector spends Claude OAuth allowance according to the tightest +// remaining budget per second until reset. Other providers keep round-robin. +// It performs no network I/O on the request path. +type QuotaAwareSelector struct { + policy config.QuotaAwareRoutingConfig + roundRobin RoundRobinSelector +} + +func NewQuotaAwareSelector(policy config.QuotaAwareRoutingConfig) *QuotaAwareSelector { + policy.Reserves = append([]config.QuotaReserve(nil), policy.Reserves...) + return &QuotaAwareSelector{policy: policy} +} + +func quotaSelector(selector Selector) *QuotaAwareSelector { + if affinity, ok := selector.(*SessionAffinitySelector); ok { + selector = affinity.fallback + } + q, _ := selector.(*QuotaAwareSelector) + return q +} + +func routingFallbackCandidates(fallback Selector, available []*Auth) []*Auth { + if quotaSelector(fallback) != nil { + return available + } + return highestPriorityAuths(available) +} + +func (q *QuotaAwareSelector) reserve(auth *Auth) float64 { + if auth == nil || auth.Provider != "claude" { + return 0 + } + email, _ := auth.Metadata["email"].(string) + for _, reserve := range q.policy.Reserves { + if reserve.Provider == auth.Provider && strings.EqualFold(strings.TrimSpace(reserve.Account), strings.TrimSpace(email)) { + return reserve.Percent + } + } + return 0 +} + +type quotaBudget struct { + score float64 + known, blocked bool +} + +func (q *QuotaAwareSelector) budget(auth *Auth, model string, now time.Time) quotaBudget { + if auth == nil || auth.Provider != "claude" || auth.AuthKind() == AuthKindAPIKey { + return quotaBudget{} + } + reserve := q.reserve(auth) + if reserve >= 100 { + return quotaBudget{blocked: true} + } + if auth.Quota.ObservedAt.IsZero() || now.Before(auth.Quota.ObservedAt) || now.Sub(auth.Quota.ObservedAt) > q.policy.MaxAgeDuration() { + return quotaBudget{blocked: reserve > 0} + } + windows := []string{"5h", "7d"} + lowerModel := strings.ToLower(model) + for _, family := range []string{"opus", "sonnet"} { + window := "7d-" + family + if strings.Contains(lowerModel, family) && auth.Quota.Signals[http.CanonicalHeaderKey("Anthropic-Ratelimit-Unified-"+window+"-Utilization")] != "" { + windows = append(windows, window) + } + } + score := math.Inf(1) + complete := true + for _, window := range windows { + prefix := "Anthropic-Ratelimit-Unified-" + window + used, errUsed := strconv.ParseFloat(auth.Quota.Signals[http.CanonicalHeaderKey(prefix+"-Utilization")], 64) + reset, errReset := strconv.ParseInt(auth.Quota.Signals[http.CanonicalHeaderKey(prefix+"-Reset")], 10, 64) + if errUsed != nil || math.IsNaN(used) || math.IsInf(used, 0) || used < 0 || errReset != nil || reset <= now.Unix() { + complete = false + continue + } + remaining := 100*(1-used) - reserve + if remaining <= 0 { + return quotaBudget{known: true, blocked: true} + } + rate := remaining / math.Max(1, time.Unix(reset, 0).Sub(now).Seconds()) + score = math.Min(score, rate) + } + if !complete { + return quotaBudget{blocked: reserve > 0} + } + return quotaBudget{score: score, known: true} +} + +func (q *QuotaAwareSelector) eligible(auths []*Auth, model string, now time.Time) ([]*Auth, error) { + eligible := make([]*Auth, 0, len(auths)) + for _, auth := range auths { + if auth != nil && !q.budget(auth, model, now).blocked { + eligible = append(eligible, auth) + } + } + if len(eligible) == 0 { + return nil, &Error{Code: "quota_reserve", Message: "no account has spendable quota; reserved or unverified allowance is protected", HTTPStatus: http.StatusServiceUnavailable} + } + return eligible, nil +} + +func (q *QuotaAwareSelector) Pick(ctx context.Context, provider, model string, opts cliproxyexecutor.Options, auths []*Auth) (*Auth, error) { + now := time.Now() + available, err := getSelectorAvailableAuthsAcrossPriorities(ctx, auths, provider, model, now) + if err != nil { + return nil, err + } + available, err = q.eligible(available, model, now) + if err != nil { + return nil, err + } + bestScore := -1.0 + var best []*Auth + for _, auth := range available { + budget := q.budget(auth, model, now) + if !budget.known { + continue + } + if budget.score > bestScore { + bestScore = budget.score + best = []*Auth{auth} + } else if budget.score == bestScore { + best = append(best, auth) + } + } + if len(best) == 0 { + best = available + } + return q.roundRobin.Pick(ctx, provider, model, opts, highestPriorityAuths(best)) +} + +// ObserveQuotaHeaders seeds the existing passive snapshot from an authenticated +// usage probe. It cannot overwrite a newer response or a replaced credential. +func (m *Manager) ObserveQuotaHeaders(snapshot *Auth, headers http.Header, observedAt time.Time) bool { + if m == nil || snapshot == nil { + return false + } + m.mu.Lock() + defer m.mu.Unlock() + current := m.auths[snapshot.ID] + if current == nil || current.Disabled || current.RegistrationEpoch != snapshot.RegistrationEpoch || current.CredentialVersion != snapshot.CredentialVersion || current.Quota.ObservedAt.After(observedAt) { + return false + } + return current.Quota.ObserveResponseHeadersForProvider(current.Provider, headers, observedAt) +} diff --git a/sdk/cliproxy/auth/selector_quota_test.go b/sdk/cliproxy/auth/selector_quota_test.go new file mode 100644 index 00000000000..e064870219b --- /dev/null +++ b/sdk/cliproxy/auth/selector_quota_test.go @@ -0,0 +1,212 @@ +package auth + +import ( + "context" + "fmt" + "net/http" + "strconv" + "testing" + "time" + + "github.com/router-for-me/CLIProxyAPI/v8/internal/config" + "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" + "github.com/router-for-me/CLIProxyAPI/v8/sdk/pluginapi" + sdktranslator "github.com/router-for-me/CLIProxyAPI/v8/sdk/translator" +) + +func quotaTestAuth(id string, now time.Time, used float64, weeklyReset time.Duration) *Auth { + h := http.Header{} + for window, reset := range map[string]time.Duration{"5h": 5 * time.Hour, "7d": weeklyReset} { + h.Set("Anthropic-Ratelimit-Unified-"+window+"-Utilization", strconv.FormatFloat(used, 'f', -1, 64)) + h.Set("Anthropic-Ratelimit-Unified-"+window+"-Reset", strconv.FormatInt(now.Add(reset).Unix(), 10)) + } + a := &Auth{ID: id, Provider: "claude", Status: StatusActive, Attributes: map[string]string{"priority": "0"}, Metadata: map[string]any{"email": id + "@example.com", "access_token": "test-token"}} + a.Quota.ObserveResponseHeadersForProvider("claude", h, now) + return a +} + +func quotaTestSelector() *QuotaAwareSelector { + return NewQuotaAwareSelector(config.QuotaAwareRoutingConfig{Reserves: []config.QuotaReserve{{Provider: "claude", Account: "personal@example.com", Percent: 20}}}) +} + +func TestQuotaAwareBudget(t *testing.T) { + now := time.Unix(1800000000, 0) + for _, tc := range []struct { + name string + change func(*Auth) + known, blocked bool + }{ + {name: "healthy", known: true}, + {name: "five hour reserve", change: func(a *Auth) { a.Quota.Signals["Anthropic-Ratelimit-Unified-5h-Utilization"] = "0.81" }, known: true, blocked: true}, + {name: "weekly reserve boundary", change: func(a *Auth) { a.Quota.Signals["Anthropic-Ratelimit-Unified-7d-Utilization"] = "0.8" }, known: true, blocked: true}, + {name: "stale", change: func(a *Auth) { a.Quota.ObservedAt = now.Add(-16 * time.Minute) }, blocked: true}, + {name: "missing weekly", change: func(a *Auth) { delete(a.Quota.Signals, "Anthropic-Ratelimit-Unified-7d-Utilization") }, blocked: true}, + {name: "invalid", change: func(a *Auth) { a.Quota.Signals["Anthropic-Ratelimit-Unified-7d-Utilization"] = "NaN" }, blocked: true}, + {name: "expired reset", change: func(a *Auth) { + a.Quota.Signals["Anthropic-Ratelimit-Unified-7d-Reset"] = strconv.FormatInt(now.Unix(), 10) + }, blocked: true}, + {name: "opus reserve", change: func(a *Auth) { + a.Quota.Signals["Anthropic-Ratelimit-Unified-7d-Opus-Utilization"] = "0.81" + a.Quota.Signals["Anthropic-Ratelimit-Unified-7d-Opus-Reset"] = strconv.FormatInt(now.Add(time.Hour).Unix(), 10) + }, known: true, blocked: true}, + } { + t.Run(tc.name, func(t *testing.T) { + a := quotaTestAuth("personal", now, 0.5, 3*24*time.Hour) + if tc.change != nil { + tc.change(a) + } + got := quotaTestSelector().budget(a, "claude-opus", now) + if got.known != tc.known || got.blocked != tc.blocked { + t.Fatalf("budget = %+v", got) + } + }) + } + unknown := quotaTestAuth("work", now, 0.5, time.Hour) + unknown.Quota = QuotaState{} + if quotaTestSelector().budget(unknown, "claude-opus", now).blocked { + t.Fatal("unprotected unknown account must remain available") + } +} + +func TestQuotaAwareRanksAcrossPriorityTiers(t *testing.T) { + now := time.Now().Add(-time.Second) + later := quotaTestAuth("later", now, 0.5, 6*24*time.Hour) + later.Attributes["priority"] = "10" + soon := quotaTestAuth("soon", now, 0.5, 24*time.Hour) + q := quotaTestSelector() + for _, affinity := range []bool{false, true} { + t.Run(fmt.Sprint(affinity), func(t *testing.T) { + var selector Selector = q + if affinity { + s := NewSessionAffinitySelector(q) + defer s.Stop() + selector = s + } + m := NewManager(nil, selector, nil) + m.RegisterExecutor(schedulerTestExecutor{provider: "claude"}) + for _, a := range []*Auth{later, soon} { + a = a.Clone() + a.ID += fmt.Sprint(affinity) + registry.GetGlobalRegistry().RegisterClient(a.ID, "claude", []*registry.ModelInfo{{ID: "claude-opus-test"}}) + t.Cleanup(func() { registry.GetGlobalRegistry().UnregisterClient(a.ID) }) + if _, err := m.Register(context.Background(), a); err != nil { + t.Fatal(err) + } + } + if m.useSchedulerFastPath() { + t.Fatal("quota routing must not use static priority fast path") + } + selected, _, err := m.pickNext(context.Background(), "claude", "claude-opus-test", cliproxyexecutor.Options{}, nil) + if err != nil || selected.ID != "soon"+fmt.Sprint(affinity) { + t.Fatalf("picked %v, %v", selected, err) + } + stored, _ := m.GetByID(selected.ID) + if stored.Attributes["priority"] != "0" { + t.Fatal("selection must not rewrite credential priorities") + } + }) + } +} + +func TestQuotaAwareReserveOverridesAffinity(t *testing.T) { + for _, mode := range []string{"explicit", "lcp"} { + t.Run(mode, func(t *testing.T) { + now := time.Now().Add(-time.Second) + personal := quotaTestAuth("personal", now, 0.1, time.Hour) + work := quotaTestAuth("work", now, 0.5, 5*24*time.Hour) + s := NewSessionAffinitySelector(quotaTestSelector()) + defer s.Stop() + opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": []string{"reserve-session"}}} + if mode == "lcp" { + opts = cliproxyexecutor.Options{SourceFormat: sdktranslator.FormatOpenAI, OriginalRequest: []byte(`{"messages":[{"role":"system","content":"stable"},{"role":"user","content":"first"}]}`), Metadata: map[string]any{cliproxyexecutor.CallerScopeMetadataKey: "reserve-caller"}} + } + auths := []*Auth{personal, work} + first, err := s.Pick(context.Background(), "claude", "claude-opus", opts, auths) + if err != nil || first.ID != "personal" { + t.Fatalf("first pick %v, %v", first, err) + } + personal.Quota.Signals["Anthropic-Ratelimit-Unified-7d-Utilization"] = "0.82" + second, err := s.Pick(context.Background(), "claude", "claude-opus", opts, auths) + if err != nil || second.ID != "work" { + t.Fatalf("reserve did not override affinity: %v, %v", second, err) + } + if _, err = s.Pick(context.Background(), "claude", "claude-opus", opts, []*Auth{personal}); err == nil { + t.Fatal("must fail rather than spend reserved allowance") + } + }) + } +} + +func TestQuotaAwareKeepsHealthyAffinityAndOtherProviderPriority(t *testing.T) { + now := time.Now().Add(-time.Second) + q := quotaTestSelector() + s := NewSessionAffinitySelector(q) + defer s.Stop() + a, b := quotaTestAuth("a", now, 0.5, time.Hour), quotaTestAuth("b", now, 0.5, 24*time.Hour) + opts := cliproxyexecutor.Options{Headers: http.Header{"Session_id": []string{"sticky"}}} + first, err := s.Pick(context.Background(), "claude", "model", opts, []*Auth{a, b}) + if err != nil { + t.Fatal(err) + } + b.Quota.Signals["Anthropic-Ratelimit-Unified-7d-Reset"] = strconv.FormatInt(now.Add(5*time.Minute).Unix(), 10) + again, err := s.Pick(context.Background(), "claude", "model", opts, []*Auth{a, b}) + if err != nil || again.ID != first.ID { + t.Fatal("healthy session must stay bound") + } + codex := []*Auth{{ID: "low", Provider: "codex", Attributes: map[string]string{"priority": "0"}}, {ID: "high", Provider: "codex", Attributes: map[string]string{"priority": "10"}}} + picked, err := q.Pick(context.Background(), "codex", "model", cliproxyexecutor.Options{}, codex) + if err != nil || picked.ID != "high" { + t.Fatalf("other provider priority changed: %v %v", picked, err) + } +} + +func TestQuotaProbeCannotOverwriteNewerOrReplacedCredential(t *testing.T) { + now := time.Now() + m := NewManager(nil, quotaTestSelector(), nil) + a, err := m.Register(context.Background(), quotaTestAuth("personal", now, 0.1, time.Hour)) + if err != nil { + t.Fatal(err) + } + h := http.Header{"Anthropic-Ratelimit-Unified-5h-Utilization": []string{"0.9"}} + if m.ObserveQuotaHeaders(a, h, now.Add(-time.Second)) { + t.Fatal("old probe replaced a newer response") + } + stale := a.Clone() + stale.CredentialVersion++ + if m.ObserveQuotaHeaders(stale, h, now.Add(time.Second)) { + t.Fatal("probe from another credential was accepted") + } + if !m.ObserveQuotaHeaders(a, h, now.Add(time.Second)) { + t.Fatal("fresh matching probe was rejected") + } +} + +func TestQuotaAwarePluginDelegateCannotBypassReserves(t *testing.T) { + now := time.Now().Add(-time.Second) + q := quotaTestSelector() + m := NewManager(nil, q, nil) + personal := quotaTestAuth("personal", now, 0.9, time.Hour) + work := quotaTestAuth("work", now, 0.2, time.Hour) + plugin := &fakePluginScheduler{handled: true, resp: pluginapi.SchedulerPickResponse{Handled: true, DelegateBuiltin: pluginapi.SchedulerBuiltinRoundRobin}} + selected, handled, err := m.pickViaPluginScheduler(context.Background(), plugin, "claude", nil, "model", cliproxyexecutor.Options{}, nil, []*Auth{work}) + if err != nil || handled || selected != nil { + t.Fatalf("plugin delegate must return to quota selector: %v, %v, %v", selected, handled, err) + } + if _, err := q.eligible([]*Auth{personal}, "model", now); err == nil { + t.Fatal("reserved account was eligible") + } +} + +func TestQuotaAwarePreservesCodexPriorityBeforeWebsocketPreference(t *testing.T) { + q := quotaTestSelector() + auths := []*Auth{ + {ID: "low-ws", Provider: "codex", Attributes: map[string]string{"priority": "0", "websockets": "true"}}, + {ID: "high-http", Provider: "codex", Attributes: map[string]string{"priority": "10", "websockets": "false"}}, + } + ctx := cliproxyexecutor.WithDownstreamWebsocket(context.Background()) + selected, err := q.Pick(ctx, "codex", "model", cliproxyexecutor.Options{}, auths) + if err != nil || selected.ID != "high-http" { + t.Fatalf("WebSocket preference bypassed priority: %v %v", selected, err) + } +} diff --git a/sdk/cliproxy/builder.go b/sdk/cliproxy/builder.go index 532bde10c59..13d434b4182 100644 --- a/sdk/cliproxy/builder.go +++ b/sdk/cliproxy/builder.go @@ -203,6 +203,9 @@ func (b *Builder) Build() (*Service, error) { if b.configPath == "" { return nil, fmt.Errorf("cliproxy: configuration path is required") } + if errValidate := b.cfg.ValidateQuotaRouting(); errValidate != nil { + return nil, errValidate + } if errValidate := b.cfg.ValidateCredentialWeights(); errValidate != nil { return nil, fmt.Errorf("cliproxy: validate credential weights: %w", errValidate) } diff --git a/sdk/cliproxy/service_config.go b/sdk/cliproxy/service_config.go index 890af119e08..6863ba34830 100644 --- a/sdk/cliproxy/service_config.go +++ b/sdk/cliproxy/service_config.go @@ -2,9 +2,11 @@ package cliproxy import ( "context" + "encoding/json" "strings" "time" + internalconfig "github.com/router-for-me/CLIProxyAPI/v8/internal/config" "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" "github.com/router-for-me/CLIProxyAPI/v8/internal/watcher/synthesizer" coreauth "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth" @@ -56,6 +58,7 @@ type configCommit struct { type routingRuntimeState struct { strategy string + quotaAwareJSON string sessionAffinity bool sessionAffinityTTL time.Duration sessionAffinitySubagents bool @@ -76,6 +79,10 @@ func normalizedRoutingRuntimeState(cfg *config.Config) routingRuntimeState { switch strings.ToLower(strings.TrimSpace(cfg.Routing.Strategy)) { case "weighted-round-robin", "weightedroundrobin", "wrr": state.strategy = "weighted-round-robin" + case "quota-aware": + state.strategy = "quota-aware" + policy, _ := json.Marshal(cfg.Routing.QuotaAware) + state.quotaAwareJSON = string(policy) case "fill-first", "fillfirst", "ff": state.strategy = "fill-first" } @@ -100,6 +107,10 @@ func normalizedRoutingRuntimeState(cfg *config.Config) routingRuntimeState { func newRoutingSelector(state routingRuntimeState) coreauth.Selector { var selector coreauth.Selector switch state.strategy { + case "quota-aware": + var policy internalconfig.QuotaAwareRoutingConfig + _ = json.Unmarshal([]byte(state.quotaAwareJSON), &policy) + selector = coreauth.NewQuotaAwareSelector(policy) case "weighted-round-robin": selector = &coreauth.WeightedRoundRobinSelector{} case "fill-first": @@ -146,6 +157,10 @@ func (s *Service) commitConfigUpdate(newCfg *config.Config) configCommit { if newCfg == nil { return configCommit{} } + if errValidate := newCfg.ValidateQuotaRouting(); errValidate != nil { + log.WithError(errValidate).Warn("rejected invalid quota routing configuration") + return configCommit{} + } if errValidate := newCfg.ValidateCredentialWeights(); errValidate != nil { log.WithError(errValidate).Warn("rejected config update with invalid credential weights") return configCommit{} diff --git a/sdk/cliproxy/service_lifecycle.go b/sdk/cliproxy/service_lifecycle.go index f08eb1b4cf3..64688b7def1 100644 --- a/sdk/cliproxy/service_lifecycle.go +++ b/sdk/cliproxy/service_lifecycle.go @@ -121,6 +121,8 @@ func (s *Service) Run(ctx context.Context) error { } } + go s.runQuotaRefresh(ctx) + // legacy clients removed; no caches to refresh s.ensureWebsocketGateway() diff --git a/sdk/cliproxy/service_quota.go b/sdk/cliproxy/service_quota.go new file mode 100644 index 00000000000..c0e9cb7670a --- /dev/null +++ b/sdk/cliproxy/service_quota.go @@ -0,0 +1,142 @@ +package cliproxy + +import ( + "context" + "encoding/json" + "fmt" + "io" + "math" + "net/http" + "strconv" + "strings" + "time" + + coreauth "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth" + log "github.com/sirupsen/logrus" +) + +type claudeUsageWindow struct { + Utilization *float64 `json:"utilization"` + ResetsAt *time.Time `json:"resets_at"` +} + +func quotaRetryBackoff(value string, now time.Time) time.Duration { + backoff := 15 * time.Minute + if seconds, errParse := strconv.Atoi(value); errParse == nil && seconds > 0 { + return max(backoff, time.Duration(seconds)*time.Second) + } + if until, errParse := http.ParseTime(value); errParse == nil { + return max(backoff, until.Sub(now)) + } + return backoff +} + +// claudeUsageHeaders converts the usage endpoint's percentages to the same +// fractional header format already observed on normal Claude responses. +func claudeUsageHeaders(body io.Reader, observedAt time.Time) (http.Header, error) { + var usage map[string]json.RawMessage + if errDecode := json.NewDecoder(io.LimitReader(body, 64<<10)).Decode(&usage); errDecode != nil { + return nil, fmt.Errorf("decode Claude usage: %w", errDecode) + } + headers := http.Header{} + for name, window := range map[string]string{"five_hour": "5h", "seven_day": "7d", "seven_day_opus": "7d-opus", "seven_day_sonnet": "7d-sonnet"} { + var w *claudeUsageWindow + if raw := usage[name]; len(raw) > 0 { + if errWindow := json.Unmarshal(raw, &w); errWindow != nil { + return nil, fmt.Errorf("decode Claude quota window %s: %w", name, errWindow) + } + } + if w == nil || w.Utilization == nil { + continue + } + used := *w.Utilization + if math.IsNaN(used) || math.IsInf(used, 0) || used < 0 { + return nil, fmt.Errorf("invalid Claude usage percentage") + } + reset := w.ResetsAt + if reset == nil { + // An unused window has no reset until its first request starts it. + if used != 0 { + return nil, fmt.Errorf("used Claude window has no reset") + } + duration := 7 * 24 * time.Hour + if name == "five_hour" { + duration = 5 * time.Hour + } + future := observedAt.Add(duration) + reset = &future + } + prefix := "Anthropic-Ratelimit-Unified-" + window + headers.Set(prefix+"-Utilization", strconv.FormatFloat(used/100, 'f', -1, 64)) + headers.Set(prefix+"-Reset", strconv.FormatInt(reset.Unix(), 10)) + } + if headers.Get("Anthropic-Ratelimit-Unified-5h-Utilization") == "" || headers.Get("Anthropic-Ratelimit-Unified-7d-Utilization") == "" { + return nil, fmt.Errorf("Claude usage is missing shared quota windows") + } + return headers, nil +} + +func (s *Service) refreshClaudeQuota(ctx context.Context, auth *coreauth.Auth) (time.Duration, error) { + startedAt := time.Now() + req, errRequest := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.anthropic.com/api/oauth/usage", nil) + if errRequest != nil { + return 0, errRequest + } + req.Header.Set("Anthropic-Beta", "oauth-2025-04-20") + req.Header.Set("Accept", "application/json") + resp, errHTTP := s.coreManager.HttpRequest(ctx, auth, req) + if errHTTP != nil { + return 0, fmt.Errorf("Claude quota request failed") + } + defer func() { + if errClose := resp.Body.Close(); errClose != nil { + log.Debug("failed to close quota response body") + } + }() + if resp.StatusCode != http.StatusOK { + backoff := time.Duration(0) + if resp.StatusCode == http.StatusTooManyRequests { + backoff = quotaRetryBackoff(resp.Header.Get("Retry-After"), time.Now()) + } + return backoff, fmt.Errorf("Claude quota request returned HTTP %d", resp.StatusCode) + } + headers, errParse := claudeUsageHeaders(resp.Body, startedAt) + if errParse != nil { + return 0, errParse + } + s.coreManager.ObserveQuotaHeaders(auth, headers, startedAt) + log.WithField("auth_id", auth.ID).Debug("refreshed Claude quota routing snapshot") + return 0, nil +} + +// runQuotaRefresh is independent of request selection. Existing response headers +// continue updating active accounts between these low-frequency idle probes. +// The service context cancels network requests during shutdown. +func (s *Service) runQuotaRefresh(ctx context.Context) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + next := map[string]time.Time{} + for { + cfg := s.currentConfig() + if cfg != nil && !cfg.Home.Enabled && strings.EqualFold(strings.TrimSpace(cfg.Routing.Strategy), "quota-aware") && s.coreManager != nil { + for _, auth := range s.coreManager.List() { + if ctx.Err() != nil { + return + } + if auth.Provider != "claude" || auth.Disabled || auth.AuthKind() == coreauth.AuthKindAPIKey || time.Now().Before(next[auth.ID]) { + continue + } + backoff, errRefresh := s.refreshClaudeQuota(ctx, auth) + if errRefresh != nil && ctx.Err() == nil { + log.WithField("auth_id", auth.ID).WithError(errRefresh).Warn("quota refresh failed; reserved accounts require a fresh snapshot") + } + next[auth.ID] = time.Now().Add(max(cfg.Routing.QuotaAware.RefreshDuration(), backoff)) + } + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} diff --git a/sdk/cliproxy/service_quota_test.go b/sdk/cliproxy/service_quota_test.go new file mode 100644 index 00000000000..687ccd1deed --- /dev/null +++ b/sdk/cliproxy/service_quota_test.go @@ -0,0 +1,67 @@ +package cliproxy + +import ( + "net/http" + "strings" + "testing" + "time" + + internalconfig "github.com/router-for-me/CLIProxyAPI/v8/internal/config" + coreauth "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth" +) + +func TestClaudeUsageHeaders(t *testing.T) { + now := time.Unix(1800000000, 0) + body := `{"five_hour":{"utilization":0,"resets_at":null},"seven_day":{"utilization":61,"resets_at":"2027-02-01T00:00:00Z"},"seven_day_opus":{"utilization":81,"resets_at":"2027-02-01T00:00:00Z"},"seven_day_sonnet":null,"other_limits":[],"extra_usage":{"is_enabled":false}}` + h, err := claudeUsageHeaders(strings.NewReader(body), now) + if err != nil { + t.Fatal(err) + } + if h.Get("Anthropic-Ratelimit-Unified-7d-Utilization") != "0.61" || h.Get("Anthropic-Ratelimit-Unified-7d-Opus-Utilization") != "0.81" { + t.Fatalf("wrong percentages: %v", h) + } + if h.Get("Anthropic-Ratelimit-Unified-5h-Reset") == "" { + t.Fatal("unused window needs a conservative initial reset") + } + for _, bad := range []string{`{}`, `{"five_hour":null}`, `{"five_hour":{"utilization":10},"seven_day":{"utilization":0}}`, `not-json`} { + if _, err := claudeUsageHeaders(strings.NewReader(bad), now); err == nil { + t.Fatalf("accepted invalid usage %q", bad) + } + } +} + +func TestQuotaAwareRoutingConfigReload(t *testing.T) { + cfg := &internalconfig.Config{Routing: internalconfig.RoutingConfig{Strategy: "quota-aware", QuotaAware: internalconfig.QuotaAwareRoutingConfig{Reserves: []internalconfig.QuotaReserve{{Provider: "claude", Account: "personal@example.com", Percent: 20}}}}} + state := normalizedRoutingRuntimeState(cfg) + if _, ok := newRoutingSelector(state).(*coreauth.QuotaAwareSelector); !ok { + t.Fatal("quota-aware strategy was not selected") + } + cfg.Routing.QuotaAware.Reserves[0].Percent = 30 + if normalizedRoutingRuntimeState(cfg) == state { + t.Fatal("reserve change must rebuild routing policy") + } + service := &Service{cfg: cfg} + invalid := cfg.CloneForRuntime() + invalid.Routing.QuotaAware.Reserves[0].Percent = 101 + if service.commitConfigUpdate(invalid).cfg != nil || service.currentConfig() != cfg { + t.Fatal("invalid reserve replaced current configuration") + } +} + +func TestQuotaRetryBackoff(t *testing.T) { + now := time.Date(2026, 10, 26, 4, 0, 0, 0, time.UTC) + for _, tc := range []struct { + value string + want time.Duration + }{ + {"3600", time.Hour}, + {now.Add(90 * time.Minute).Format(http.TimeFormat), 90 * time.Minute}, + {"5", 15 * time.Minute}, + {now.Add(-time.Hour).Format(http.TimeFormat), 15 * time.Minute}, + {"invalid", 15 * time.Minute}, + } { + if got := quotaRetryBackoff(tc.value, now); got != tc.want { + t.Fatalf("Retry-After %q = %v, want %v", tc.value, got, tc.want) + } + } +}