diff --git a/.gitignore b/.gitignore index e1961644d0..c24ff2f978 100755 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,8 @@ nul /.local/ # Docs children ignored generally; ADRs are product docs and stay trackable /docs/* +!/docs/prometheus/ +!/docs/grafana/ !/docs/adr/ /docs/adr/* !/docs/adr/*.md diff --git a/docs/CLI.md b/docs/CLI.md index a9c25bcdcc..f0c14816ba 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -101,6 +101,8 @@ curl.exe -H "Authorization: Bearer $env:CODEXBAR_DASHBOARD_TOKEN" http://127.0.0 The metrics contract exports collection health for every enabled, known provider. The only provider label is its bounded canonical CLI slug, for example `codexbar_provider_up{provider="claude"}`; disabled providers are absent, and an ordinary provider fetch failure does not suppress healthy provider series. Quota semantics are currently exported only for Codex through fixed `session`, `weekly`, `monthly`, and `code_review` metric families. Used and remaining values are ratios from `0` to `1`, with no dynamic window label. Available local Codex cost estimates are also exported. +Codex OAuth reset credits are exposed as `codexbar_reset_credits_available{provider="codex"}` (`0` exhausted, `-1` unavailable or PAT) and, when present, `codexbar_reset_credits_next_expiry_timestamp_seconds{provider="codex"}`. Successful and unavailable reset-credit observations are cached for ten minutes. See [Prometheus examples](prometheus/README.zh-CN.md) for a Chinese Grafana dashboard, scrape settings, and alert rules. + Unknown, informational, non-finite, and dynamic additional-limit values are omitted instead of being inferred or replaced with sentinels. Account identity, display labels, source names, free-form provider errors, and version strings are not exposed. Consumers should alert on provider health, snapshot staleness, and quota values together. ### Config diff --git a/docs/grafana/codexbar-codex-zh.json b/docs/grafana/codexbar-codex-zh.json new file mode 100644 index 0000000000..1f63bcde23 --- /dev/null +++ b/docs/grafana/codexbar-codex-zh.json @@ -0,0 +1,504 @@ +{ + "__inputs": [], + "__requires": [ + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "uid": null, + "id": null, + "title": "CodexBar · Codex 监控", + "tags": [ + "codexbar", + "codex" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "1m", + "time": { + "from": "now-24h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "datasource", + "label": "Prometheus 数据源", + "type": "datasource", + "query": "prometheus", + "refresh": 1, + "current": {} + }, + { + "name": "job", + "label": "采集任务", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": { + "query": "label_values(codexbar_provider_up{provider=\"codex\"},job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "selected": true, + "text": "全部", + "value": "$__all" + } + }, + { + "name": "instance", + "label": "节点", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": { + "query": "label_values(codexbar_provider_up{provider=\"codex\",job=~\"${job:regex}\"},instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "selected": true, + "text": "全部", + "value": "$__all" + } + }, + { + "name": "window", + "label": "额度窗口", + "type": "custom", + "query": "weekly,session,monthly,code_review", + "current": { + "selected": true, + "text": "weekly", + "value": "weekly" + }, + "includeAll": false, + "multi": false, + "options": [ + { + "selected": true, + "text": "weekly", + "value": "weekly" + }, + { + "selected": false, + "text": "session", + "value": "session" + }, + { + "selected": false, + "text": "monthly", + "value": "monthly" + }, + { + "selected": false, + "text": "code_review", + "value": "code_review" + } + ] + } + ] + }, + "panels": [ + { + "id": 1, + "title": "节点采集状态", + "type": "stat", + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "codexbar_provider_up{job=~\"${job:regex}\",instance=~\"${instance:regex}\",provider=\"codex\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": {}, + "description": "1 为正常，0 为查询失败" + }, + { + "id": 2, + "title": "剩余额度", + "type": "gauge", + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "codexbar_quota_${window:raw}_remaining_ratio{job=~\"${job:regex}\",instance=~\"${instance:regex}\",provider=\"codex\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": {}, + "description": "" + }, + { + "id": 3, + "title": "重置卡可用张数", + "type": "stat", + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "codexbar_reset_credits_available{job=~\"${job:regex}\",instance=~\"${instance:regex}\",provider=\"codex\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "无法获取", + "color": "orange" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": {}, + "description": "-1 表示 PAT 模式或查询不可用；0 表示用完" + }, + { + "id": 4, + "title": "最早到期重置卡", + "type": "stat", + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 5 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "codexbar_reset_credits_next_expiry_timestamp_seconds{job=~\"${job:regex}\",instance=~\"${instance:regex}\",provider=\"codex\"} * 1000", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "dateTimeAsLocal", + "color": { + "mode": "palette-classic" + }, + "noValue": "无有效期数据" + }, + "overrides": [] + }, + "options": {}, + "description": "仅在有可用卡且接口返回有效期时显示" + }, + { + "id": 5, + "title": "剩余额度历史", + "type": "timeseries", + "gridPos": { + "h": 9, + "w": 16, + "x": 8, + "y": 5 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "codexbar_quota_${window:raw}_remaining_ratio{job=~\"${job:regex}\",instance=~\"${instance:regex}\",provider=\"codex\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": {}, + "description": "" + }, + { + "id": 6, + "title": "额度重置时间", + "type": "stat", + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 10 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "codexbar_quota_${window:raw}_reset_timestamp_seconds{job=~\"${job:regex}\",instance=~\"${instance:regex}\",provider=\"codex\"} * 1000", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "dateTimeAsLocal", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": {}, + "description": "" + }, + { + "id": 7, + "type": "row", + "title": "辅助信息", + "collapsed": true, + "gridPos": { + "x": 0, + "y": 14, + "w": 24, + "h": 1 + }, + "panels": [ + { + "id": 8, + "title": "Codex 本地估算成本（今日）", + "type": "stat", + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 15 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "codexbar_cost_today_usd{job=~\"${job:regex}\",instance=~\"${instance:regex}\",provider=\"codex\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "currencyUSD", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": {}, + "description": "" + }, + { + "id": 9, + "title": "采集数据年龄", + "type": "stat", + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 15 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "codexbar_provider_data_age_seconds{job=~\"${job:regex}\",instance=~\"${instance:regex}\",provider=\"codex\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": {}, + "description": "" + }, + { + "id": 10, + "title": "导出器状态", + "type": "stat", + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 15 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "codexbar_up{job=~\"${job:regex}\",instance=~\"${instance:regex}\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": {}, + "description": "" + } + ] + } + ], + "annotations": { + "list": [] + } +} diff --git a/docs/prometheus/README.zh-CN.md b/docs/prometheus/README.zh-CN.md new file mode 100644 index 0000000000..6c315f1af7 --- /dev/null +++ b/docs/prometheus/README.zh-CN.md @@ -0,0 +1,18 @@ +# Codex Prometheus 监控示例 + +用同一 Windows 用户运行 Codex CLI 和 `codexbar`，确保能读取该用户的 OAuth 凭据。将 `YOUR_RANDOM_TOKEN` 替换为长随机令牌、`YOUR_LAN_IP` 替换为节点实际局域网 IP 后启动： + +```cmd +set CODEXBAR_DASHBOARD_TOKEN=YOUR_RANDOM_TOKEN +codexbar.exe serve --host YOUR_LAN_IP --port 8081 --metrics --allow-plain-http +``` + +仅允许 Prometheus 主机访问该端口，并在可信局域网中传输令牌。`/metrics` 默认关闭，启用后与 `/usage` 共用 Bearer 认证。`--host` 应设为节点自身的局域网 IP，Prometheus target 应指向该 IP。验证： + +```cmd +curl.exe -H "Authorization: Bearer %CODEXBAR_DASHBOARD_TOKEN%" http://YOUR_LAN_IP:8081/metrics +``` + +将 [抓取示例](codexbar-scrape-example.yml)中的示例地址替换成实际节点 IP，并把 `scrape_configs` 和 [告警规则](codexbar-alerts.yml)的路径加入现有 `prometheus.yml`；示例规则默认匹配 `job="codexbar"`。多台节点共用一个 job 时必须使用同一个 Bearer Token；若令牌不同，则为每台节点建一个单独 job，并同步调整告警规则中的 job 选择器。运行 `promtool check config` 和 `promtool check rules` 后重载 Prometheus。导入 [Grafana 中文面板](../grafana/codexbar-codex-zh.json)，选择 Prometheus 数据源；额度窗口默认 weekly，节点可选单台或全部。 + +`codexbar_reset_credits_available{provider="codex"}` 中，`0` 为已用完，`-1` 为 PAT 模式或接口不可用。`codexbar_reset_credits_next_expiry_timestamp_seconds` 只在有可用卡且存在未来有效期时输出，不暴露卡 ID。重置卡接口查询成功或失败均缓存 10 分钟；异常的周额度重置确认会获取独立观察。配额指标以 0 到 1 的比例表示，Grafana 面板会按百分比显示。成本来自本地日志估算，不是订阅账单。 diff --git a/docs/prometheus/codexbar-alerts.yml b/docs/prometheus/codexbar-alerts.yml new file mode 100644 index 0000000000..2c1e13bd6d --- /dev/null +++ b/docs/prometheus/codexbar-alerts.yml @@ -0,0 +1,101 @@ +groups: + - name: win-codexbar-codex + rules: + - alert: CodexBarExporterDown + expr: up{job="codexbar"} == 0 + for: 10m + labels: + severity: critical + annotations: + summary: "CodexBar 节点无法抓取" + description: "{{ $labels.instance }} 连续 10 分钟无法抓取，请检查服务、网络和 Bearer Token。" + + - alert: CodexBarCollectionStale + expr: codexbar_snapshot_stale{job="codexbar"} == 1 + for: 10m + labels: + severity: warning + annotations: + summary: "CodexBar 采集数据过期" + description: "{{ $labels.instance }} 的最近一次采集已超过过期阈值。" + + - alert: CodexBarCollectionUnavailable + expr: codexbar_up{job="codexbar"} == 0 + for: 10m + labels: + severity: critical + annotations: + summary: "CodexBar 尚无可导出的采集数据" + description: "{{ $labels.instance }} 的 exporter 可抓取，但持续没有成功采集数据。" + + - alert: CodexBarCodexFetchFailed + expr: codexbar_provider_up{job="codexbar",provider="codex"} == 0 + for: 10m + labels: + severity: critical + annotations: + summary: "Codex 用量查询失败" + description: "{{ $labels.instance }} 连续 10 分钟无法取得 Codex 用量。" + + - alert: CodexBarCodexQuotaMissing + expr: | + (codexbar_provider_up{job="codexbar",provider="codex"} == 1) + unless on (job, instance, provider) + (codexbar_quota_weekly_remaining_ratio{job="codexbar",provider="codex"}) + for: 15m + labels: + severity: warning + annotations: + summary: "Codex 周额度数据缺失" + description: "{{ $labels.instance }} 的用量查询成功，但周额度指标持续缺失。" + + - alert: CodexBarCodexWeeklyLow + expr: | + (codexbar_quota_weekly_remaining_ratio{job="codexbar",provider="codex"} <= 0.2) + and on (job, instance, provider) + (codexbar_quota_weekly_remaining_ratio{job="codexbar",provider="codex"} > 0.05) + for: 10m + labels: + severity: warning + annotations: + summary: "Codex 周剩余额度低于 20%" + description: "{{ $labels.instance }} 当前周剩余额度为 {{ $value | humanizePercentage }}。" + + - alert: CodexBarCodexWeeklyCritical + expr: codexbar_quota_weekly_remaining_ratio{job="codexbar",provider="codex"} <= 0.05 + for: 5m + labels: + severity: critical + annotations: + summary: "Codex 周剩余额度低于 5%" + description: "{{ $labels.instance }} 当前周额度即将耗尽。" + + - alert: CodexBarResetCreditsExhausted + expr: codexbar_reset_credits_available{job="codexbar",provider="codex"} == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Codex 重置卡已用完" + description: "{{ $labels.instance }} 当前没有可用重置卡。" + + - alert: CodexBarResetCreditsUnavailable + expr: codexbar_reset_credits_available{job="codexbar",provider="codex"} == -1 + for: 15m + labels: + severity: warning + annotations: + summary: "Codex 重置卡数量无法获取" + description: "{{ $labels.instance }} 处于 PAT 模式，或重置卡接口暂不可用。" + + - alert: CodexBarResetCreditExpiringSoon + expr: | + (codexbar_reset_credits_next_expiry_timestamp_seconds{job="codexbar",provider="codex"} - time() < 86400) + and on (job, instance, provider) + (codexbar_reset_credits_available{job="codexbar",provider="codex"} > 0) + for: 5m + labels: + severity: warning + annotations: + summary: "Codex 重置卡即将到期" + description: "{{ $labels.instance }} 最早到期的可用重置卡将在 24 小时内到期。" diff --git a/docs/prometheus/codexbar-scrape-example.yml b/docs/prometheus/codexbar-scrape-example.yml new file mode 100644 index 0000000000..ca9dfc7f90 --- /dev/null +++ b/docs/prometheus/codexbar-scrape-example.yml @@ -0,0 +1,12 @@ +scrape_configs: + - job_name: codexbar + scrape_interval: 2m + metrics_path: /metrics + authorization: + type: Bearer + credentials_file: /etc/prometheus/secrets/codexbar.token + static_configs: + - targets: + - 192.0.2.11:8081 + - 192.0.2.12:8081 + - 192.0.2.13:8081 diff --git a/rust/src/cli/serve/metrics/definitions.rs b/rust/src/cli/serve/metrics/definitions.rs index 364a10bb15..4e37b421f0 100644 --- a/rust/src/cli/serve/metrics/definitions.rs +++ b/rust/src/cli/serve/metrics/definitions.rs @@ -93,6 +93,14 @@ pub(in crate::cli::serve::metrics) const METRIC_DEFINITIONS: &[(&str, &str)] = & "codexbar_cost_today_usd", "Available local Codex cost today in US dollars.", ), + ( + "codexbar_reset_credits_available", + "Available Codex reset credits; -1 means unavailable or unsupported.", + ), + ( + "codexbar_reset_credits_next_expiry_timestamp_seconds", + "Unix timestamp of the earliest expiry among available Codex reset credits.", + ), ( "codexbar_cost_last_30_days_usd", "Available local Codex cost over the last 30 days in US dollars.", diff --git a/rust/src/cli/serve/metrics/rendering.rs b/rust/src/cli/serve/metrics/rendering.rs index 4ab055f93c..05e33b7777 100644 --- a/rust/src/cli/serve/metrics/rendering.rs +++ b/rust/src/cli/serve/metrics/rendering.rs @@ -107,6 +107,19 @@ pub(in crate::cli::serve::metrics) fn render_at( QuotaMetricNames::CODE_REVIEW, )?; } + if let Some(count) = provider.reset_credits_available { + writer.sample("codexbar_reset_credits_available", &provider_labels, count)?; + } + if let Some(expiry) = provider + .reset_credits_next_expiry + .filter(|expiry| *expiry > now) + { + writer.sample( + "codexbar_reset_credits_next_expiry_timestamp_seconds", + &provider_labels, + expiry.timestamp(), + )?; + } if let Some(value) = provider.cost_today_usd { writer.sample_f64("codexbar_cost_today_usd", &provider_labels, value)?; } diff --git a/rust/src/cli/serve/metrics/snapshot.rs b/rust/src/cli/serve/metrics/snapshot.rs index 7eebaf5298..4f66841480 100644 --- a/rust/src/cli/serve/metrics/snapshot.rs +++ b/rust/src/cli/serve/metrics/snapshot.rs @@ -22,6 +22,8 @@ pub(in crate::cli::serve::metrics) struct ProviderMetrics { pub(in crate::cli::serve::metrics) up: bool, pub(in crate::cli::serve::metrics) updated_at: Option>, pub(in crate::cli::serve::metrics) codex_quota: Option, + pub(in crate::cli::serve::metrics) reset_credits_available: Option, + pub(in crate::cli::serve::metrics) reset_credits_next_expiry: Option>, pub(in crate::cli::serve::metrics) cost_today_usd: Option, pub(in crate::cli::serve::metrics) cost_last_30_days_usd: Option, } @@ -85,6 +87,25 @@ impl MetricsSnapshot { }), code_review: result.usage.code_review_window().and_then(quota_metric), }), + reset_credits_available: (id == ProviderId::Codex).then(|| { + result + .inventory + .iter() + .find(|item| item.id == "reset-credits") + .map(|item| i64::from(item.available_count)) + .unwrap_or(-1) + }), + reset_credits_next_expiry: (id == ProviderId::Codex) + .then(|| { + result + .inventory + .iter() + .find(|item| item.id == "reset-credits") + }) + .flatten() + .filter(|item| item.available_count > 0) + .and_then(|item| item.next_expires_at) + .filter(|expiry| *expiry > input.generated_at), cost_today_usd, cost_last_30_days_usd, }, @@ -93,6 +114,8 @@ impl MetricsSnapshot { up: false, updated_at: None, codex_quota: None, + reset_credits_available: None, + reset_credits_next_expiry: None, cost_today_usd, cost_last_30_days_usd, }, diff --git a/rust/src/cli/serve/metrics/tests.rs b/rust/src/cli/serve/metrics/tests.rs index e189154833..6caf7f2ebf 100644 --- a/rust/src/cli/serve/metrics/tests.rs +++ b/rust/src/cli/serve/metrics/tests.rs @@ -10,7 +10,9 @@ use crate::cli::serve::collection::{ProviderFetchEnvelope, RawCostPayload, Snaps use crate::cli::serve::dashboard; use crate::cli::serve::dashboard::coordinator::{SnapshotArtifacts, SnapshotArtifactsBuildFn}; use crate::cli::serve::dashboard::snapshot::{DashboardIdentity, SnapshotInput, build_snapshot}; -use crate::core::{NamedRateWindow, ProviderFetchResult, RateWindow, UsageSnapshot}; +use crate::core::{ + NamedRateWindow, ProviderFetchResult, ProviderInventoryItem, RateWindow, UsageSnapshot, +}; use crate::providers::codex::CodexApi; fn at(hour: u32) -> chrono::DateTime { @@ -198,6 +200,71 @@ fn renders_enabled_provider_health_and_codex_metrics_without_private_text() { } } +#[test] +fn reset_credits_export_count_and_only_future_expiry() { + let result = ProviderFetchResult::new(usage(RateWindow::new(10.0)), "oauth") + .with_inventory_item(ProviderInventoryItem { + id: "reset-credits".into(), + title: "Sensitive card title".into(), + available_count: 2, + next_expires_at: Some(at(5)), + }); + let snapshot = metrics_snapshot(vec![provider("codex", Ok(result))], HashMap::new()); + let body = render_at(&snapshot, at(2)).unwrap(); + assert!(body.contains("codexbar_reset_credits_available{provider=\"codex\"} 2\n")); + assert!( + body.contains("codexbar_reset_credits_next_expiry_timestamp_seconds{provider=\"codex\"}") + ); + assert!(!body.contains("Sensitive card title")); + let expired = render_at(&snapshot, at(6)).unwrap(); + assert_eq!( + samples( + &expired, + "codexbar_reset_credits_next_expiry_timestamp_seconds" + ) + .count(), + 0 + ); + + let empty = ProviderFetchResult::new(usage(RateWindow::new(10.0)), "oauth") + .with_inventory_item(ProviderInventoryItem { + id: "reset-credits".into(), + title: "Reset credits".into(), + available_count: 0, + next_expires_at: Some(at(5)), + }); + let empty_body = render_at( + &metrics_snapshot(vec![provider("codex", Ok(empty))], HashMap::new()), + at(2), + ) + .unwrap(); + assert!(empty_body.contains("codexbar_reset_credits_available{provider=\"codex\"} 0\n")); + assert_eq!( + samples( + &empty_body, + "codexbar_reset_credits_next_expiry_timestamp_seconds" + ) + .count(), + 0 + ); + + let unknown = render_at( + &metrics_snapshot( + vec![provider( + "codex", + Ok(ProviderFetchResult::new( + usage(RateWindow::new(10.0)), + "pat", + )), + )], + HashMap::new(), + ), + at(2), + ) + .unwrap(); + assert!(unknown.contains("codexbar_reset_credits_available{provider=\"codex\"} -1\n")); +} + #[test] fn exports_only_fixed_quota_slots_and_omits_extras() { let mut provider_usage = usage(RateWindow::no_active_session()); diff --git a/rust/src/providers/codex/api.rs b/rust/src/providers/codex/api.rs index 6ce382deab..6193491785 100755 --- a/rust/src/providers/codex/api.rs +++ b/rust/src/providers/codex/api.rs @@ -10,9 +10,11 @@ use crate::providers::openai::OpenAISubscriptionFetchResult; use base64::Engine; use chrono::{DateTime, TimeZone, Utc}; use serde::Deserialize; +use std::collections::HashMap; use std::path::PathBuf; -use std::sync::{Mutex, OnceLock}; +use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant, SystemTime}; +use tokio::sync::Mutex as AsyncMutex; #[path = "subscription.rs"] mod subscription; @@ -21,9 +23,19 @@ const DEFAULT_BASE_URL: &str = "https://chatgpt.com/backend-api"; const USAGE_PATH: &str = "/wham/usage"; const RESET_CREDITS_PATH: &str = "/wham/rate-limit-reset-credits"; const CREDENTIAL_CACHE_TTL: Duration = Duration::from_secs(5); +const RESET_CREDITS_CACHE_TTL: Duration = Duration::from_secs(600); const EXTERNAL_OAUTH_REFRESH_WINDOW: chrono::TimeDelta = chrono::Duration::minutes(5); static CREDENTIAL_CACHE: OnceLock>> = OnceLock::new(); +static RESET_CREDITS_CACHE: OnceLock>>>> = + OnceLock::new(); + +#[derive(Default)] +struct ResetCreditsCache { + loaded_at: Option, + value: Option, + confirmation_failure_at: Option, +} /// Codex API client pub struct CodexApi { @@ -113,6 +125,21 @@ impl CodexApi { pub async fn fetch_usage( &self, ) -> Result<(UsageSnapshot, Option, Option), ProviderError> { + let (usage, cost, account, _) = self.fetch_usage_with_reset_credits().await?; + Ok((usage, cost, account)) + } + + pub(super) async fn fetch_usage_with_reset_credits( + &self, + ) -> Result< + ( + UsageSnapshot, + Option, + Option, + Option, + ), + ProviderError, + > { let creds = self.load_credentials()?; let base_url = self.resolve_base_url(); let auth_path = self.get_auth_path(); @@ -120,8 +147,10 @@ impl CodexApi { let exact_oauth = creds.is_external_oauth; let mut state = weekly_reset::load(&scope); + let first_fetch_started = Instant::now(); let (first_usage, first_cost, first_credits) = self.fetch_usage_once(&creds, &base_url).await?; + let mut displayed_credits = first_credits.clone(); let observed_at = Utc::now(); let first_inventory = weekly_reset::inventory(first_credits.as_ref(), observed_at); let (usage, cost) = match weekly_reset::initial_decision( @@ -142,39 +171,55 @@ impl CodexApi { (usage, first_cost) } weekly_reset::InitialDecision::RequiresConfirmation => { + // Confirmation must compare independent inventory observations, + // even when the ordinary 10-minute cache is still fresh. + let initial_credits = self + .fresh_reset_credits_for_confirmation(&creds, &base_url, first_fetch_started) + .await; let confirmation = self.fetch_usage_once(&creds, &base_url).await; - let (confirmation_usage, confirmation_cost, confirmation_credits) = - match confirmation { - Ok(value) => value, - Err(error) => { - tracing::debug!( - %error, - "Codex weekly reset confirmation failed; preserving first successful usage" - ); - let result = Self::preserve_after_confirmation_failure( - &state, - first_usage, - first_cost, - ); - weekly_reset::save(&scope, &state); - let (usage, cost) = result; - let usage = self - .enrich_subscription_metadata( - &base_url, - &creds.access_token, - creds.account_id.as_deref(), - usage, - ) - .await; - return Ok((usage, cost, creds.account_id.clone())); - } - }; + let (confirmation_usage, confirmation_cost, _) = match confirmation { + Ok(value) => value, + Err(error) => { + tracing::debug!( + %error, + "Codex weekly reset confirmation failed; preserving first successful usage" + ); + let result = Self::preserve_after_confirmation_failure( + &state, + first_usage, + first_cost, + ); + weekly_reset::save(&scope, &state); + let (usage, cost) = result; + let usage = self + .enrich_subscription_metadata( + &base_url, + &creds.access_token, + creds.account_id.as_deref(), + usage, + ) + .await; + return Ok((usage, cost, creds.account_id.clone(), displayed_credits)); + } + }; + let confirmation_credits = if initial_credits.is_some() { + self.fetch_rate_limit_reset_credits_fresh(&creds, &base_url) + .await + } else { + None + }; + displayed_credits = confirmation_credits + .clone() + .or(initial_credits.clone()) + .or(displayed_credits); let confirmation_inventory = weekly_reset::inventory(confirmation_credits.as_ref(), Utc::now()); + let initial_inventory = + weekly_reset::inventory(initial_credits.as_ref(), observed_at); match weekly_reset::confirmation_decision( &mut state, &first_usage, - first_inventory.as_ref(), + initial_inventory.as_ref(), &confirmation_usage, confirmation_inventory.as_ref(), exact_oauth, @@ -205,7 +250,8 @@ impl CodexApi { usage, ) .await; - Ok((usage, cost, creds.account_id.clone())) + let usage = apply_reset_credits_window(usage, displayed_credits.as_ref()); + Ok((usage, cost, creds.account_id.clone(), displayed_credits)) } /// Subscription metadata is optional enrichment. Usage remains usable when @@ -271,16 +317,111 @@ impl CodexApi { .map_err(|e| ProviderError::Parse(e.to_string()))?; let (mut usage, cost) = self.build_result_from_json(&json)?; let reset_credits = self + .fetch_rate_limit_reset_credits_cached(creds, base_url) + .await; + usage = apply_reset_credits_window(usage, reset_credits.as_ref()); + Ok((usage, cost, reset_credits)) + } + + fn reset_credits_cache_slot( + &self, + creds: &CodexCredentials, + base_url: &str, + ) -> Arc> { + let auth_path = self.get_auth_path(); + let account = weekly_reset::scope_key(creds.account_id.as_deref(), &auth_path); + let token = weekly_reset::scope_key(Some(&creds.access_token), &auth_path); + let key = format!("{}|{account}|{token}", base_url.trim_end_matches('/')); + let cache = RESET_CREDITS_CACHE.get_or_init(|| Mutex::new(HashMap::new())); + let mut cache = cache + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + Arc::clone( + cache + .entry(key) + .or_insert_with(|| Arc::new(AsyncMutex::new(ResetCreditsCache::default()))), + ) + } + + async fn fetch_rate_limit_reset_credits_cached( + &self, + creds: &CodexCredentials, + base_url: &str, + ) -> Option { + let slot = self.reset_credits_cache_slot(creds, base_url); + let mut cache = slot.lock().await; + if cache + .loaded_at + .is_some_and(|loaded| loaded.elapsed() < RESET_CREDITS_CACHE_TTL) + { + return cache.value.clone(); + } + cache.value = self .fetch_rate_limit_reset_credits(creds, base_url) .await .ok(); - if let Some(reset_credits) = reset_credits.as_ref() - && reset_credits.available_count > 0 + cache.loaded_at = Some(Instant::now()); + cache.confirmation_failure_at = None; + cache.value.clone() + } + + async fn fresh_reset_credits_for_confirmation( + &self, + creds: &CodexCredentials, + base_url: &str, + started: Instant, + ) -> Option { + let slot = self.reset_credits_cache_slot(creds, base_url); + let mut cache = slot.lock().await; + cache.value.as_ref()?; + if cache + .confirmation_failure_at + .is_some_and(|failed| failed.elapsed() < RESET_CREDITS_CACHE_TTL) { - let window = reset_credits_rate_window(reset_credits, Utc::now()); - usage = usage.with_extra_rate_window("reset-credits", "Reset credits", window); + return None; } - Ok((usage, cost, reset_credits)) + if cache.loaded_at.is_some_and(|loaded| loaded >= started) { + return cache.value.clone(); + } + let fresh = self + .fetch_rate_limit_reset_credits(creds, base_url) + .await + .ok(); + if let Some(value) = fresh.as_ref() { + cache.value = Some(value.clone()); + cache.loaded_at = Some(Instant::now()); + cache.confirmation_failure_at = None; + } else { + cache.confirmation_failure_at = Some(Instant::now()); + } + fresh + } + + async fn fetch_rate_limit_reset_credits_fresh( + &self, + creds: &CodexCredentials, + base_url: &str, + ) -> Option { + let slot = self.reset_credits_cache_slot(creds, base_url); + let mut cache = slot.lock().await; + if cache + .confirmation_failure_at + .is_some_and(|failed| failed.elapsed() < RESET_CREDITS_CACHE_TTL) + { + return None; + } + let fresh = self + .fetch_rate_limit_reset_credits(creds, base_url) + .await + .ok(); + if let Some(value) = fresh.as_ref() { + cache.value = Some(value.clone()); + cache.loaded_at = Some(Instant::now()); + cache.confirmation_failure_at = None; + } else { + cache.confirmation_failure_at = Some(Instant::now()); + } + fresh } async fn fetch_rate_limit_reset_credits( @@ -1068,7 +1209,6 @@ pub(super) struct ResetCredit { pub(super) struct ResetCredits { #[serde(default)] pub(super) credits: Vec, - #[serde(default)] pub(super) available_count: u32, } @@ -1090,7 +1230,7 @@ fn is_available_credit(credit: &ResetCredit) -> bool { } } -fn next_available_reset_credit_expiry( +pub(super) fn next_available_reset_credit_expiry( credits: &[ResetCredit], now: DateTime, ) -> Option> { @@ -1113,6 +1253,20 @@ fn reset_credits_rate_window(reset: &ResetCredits, now: DateTime) -> RateWi window } +fn apply_reset_credits_window( + mut usage: UsageSnapshot, + reset: Option<&ResetCredits>, +) -> UsageSnapshot { + usage + .extra_rate_windows + .retain(|window| window.id != "reset-credits"); + if let Some(reset) = reset.filter(|reset| reset.available_count > 0) { + let window = reset_credits_rate_window(reset, Utc::now()); + usage = usage.with_extra_rate_window("reset-credits", "Reset credits", window); + } + usage +} + impl CreditDetails { // Helper to safely check has_credits fn has_credits(&self) -> bool { @@ -1405,6 +1559,11 @@ mod tests { ); } + #[test] + fn missing_reset_credit_count_is_unavailable_not_zero() { + assert!(decode_reset_credits(br#"{"credits":[]}"#).is_err()); + } + #[test] fn next_expiry_picks_soonest_available() { let now = DateTime::parse_from_rfc3339("2026-07-01T00:00:00Z") @@ -1550,6 +1709,161 @@ mod tests { dir } + #[tokio::test] + async fn reset_credit_cache_single_flight_and_unknown_are_ten_minute_observations() { + let mut server = mockito::Server::new_async().await; + let request = server + .mock("GET", "/wham/rate-limit-reset-credits") + .expect(1) + .with_status(503) + .create_async() + .await; + let home = write_codex_home(&server.url()); + let api = CodexApi::new().with_codex_home(home.path()); + let creds = api.load_credentials().unwrap(); + let base = server.url(); + let (first, second) = tokio::join!( + api.fetch_rate_limit_reset_credits_cached(&creds, &base), + api.fetch_rate_limit_reset_credits_cached(&creds, &base), + ); + assert!(first.is_none() && second.is_none()); + assert!( + api.fetch_rate_limit_reset_credits_cached(&creds, &base) + .await + .is_none() + ); + request.assert_async().await; + assert!(RESET_CREDITS_CACHE_TTL == Duration::from_secs(600)); + } + + #[tokio::test] + async fn reset_credit_cache_expires_and_token_rotation_uses_new_scope() { + let mut server = mockito::Server::new_async().await; + let first = server + .mock("GET", "/wham/rate-limit-reset-credits") + .match_header("authorization", "Bearer test-token") + .expect(1) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"available_count":2,"credits":[]}"#) + .create_async() + .await; + let home = write_codex_home(&server.url()); + let api = CodexApi::new().with_codex_home(home.path()); + let mut creds = api.load_credentials().unwrap(); + let base = server.url(); + assert_eq!( + api.fetch_rate_limit_reset_credits_cached(&creds, &base) + .await + .unwrap() + .available_count, + 2 + ); + assert_eq!( + api.fetch_rate_limit_reset_credits_cached(&creds, &base) + .await + .unwrap() + .available_count, + 2 + ); + first.assert_async().await; + first.remove_async().await; + let second = server + .mock("GET", "/wham/rate-limit-reset-credits") + .match_header("authorization", "Bearer test-token") + .expect(1) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"available_count":1,"credits":[]}"#) + .create_async() + .await; + api.reset_credits_cache_slot(&creds, &base) + .lock() + .await + .loaded_at = Some(Instant::now() - RESET_CREDITS_CACHE_TTL); + assert_eq!( + api.fetch_rate_limit_reset_credits_cached(&creds, &base) + .await + .unwrap() + .available_count, + 1 + ); + second.assert_async().await; + second.remove_async().await; + creds.access_token = "rotated-token".into(); + let rotated = server + .mock("GET", "/wham/rate-limit-reset-credits") + .match_header("authorization", "Bearer rotated-token") + .expect(1) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"available_count":3,"credits":[]}"#) + .create_async() + .await; + assert_eq!( + api.fetch_rate_limit_reset_credits_cached(&creds, &base) + .await + .unwrap() + .available_count, + 3 + ); + rotated.assert_async().await; + } + + #[tokio::test] + async fn suspicious_weekly_reset_uses_independent_credit_observations() { + let mut server = mockito::Server::new_async().await; + let cached_response = server + .mock("GET", "/wham/rate-limit-reset-credits") + .expect(1) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"available_count":2,"credits":[]}"#) + .create_async() + .await; + let home = write_codex_home(&server.url()); + let api = CodexApi::new().with_codex_home(home.path()); + let creds = api.load_credentials().unwrap(); + let base = server.url(); + api.fetch_rate_limit_reset_credits_cached(&creds, &base) + .await + .unwrap(); + cached_response.assert_async().await; + cached_response.remove_async().await; + + let started = Instant::now(); + let initial_response = server + .mock("GET", "/wham/rate-limit-reset-credits") + .expect(1) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"available_count":1,"credits":[]}"#) + .create_async() + .await; + let initial = api + .fresh_reset_credits_for_confirmation(&creds, &base, started) + .await + .unwrap(); + assert_eq!(initial.available_count, 1); + initial_response.assert_async().await; + initial_response.remove_async().await; + + let confirmation_response = server + .mock("GET", "/wham/rate-limit-reset-credits") + .expect(1) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"available_count":0,"credits":[]}"#) + .create_async() + .await; + let confirmation = api + .fetch_rate_limit_reset_credits_fresh(&creds, &base) + .await + .unwrap(); + assert_eq!(confirmation.available_count, 0); + confirmation_response.assert_async().await; + } + #[tokio::test] async fn fetch_usage_attaches_reset_credits_from_http() { let mut server = mockito::Server::new_async().await; diff --git a/rust/src/providers/codex/mod.rs b/rust/src/providers/codex/mod.rs index 99175da309..029f6dc688 100755 --- a/rust/src/providers/codex/mod.rs +++ b/rust/src/providers/codex/mod.rs @@ -12,8 +12,8 @@ use async_trait::async_trait; use std::os::windows::process::CommandExt; use crate::core::{ - FetchContext, Provider, ProviderError, ProviderFetchResult, ProviderId, ProviderMetadata, - SourceMode, + FetchContext, Provider, ProviderError, ProviderFetchResult, ProviderId, ProviderInventoryItem, + ProviderMetadata, SourceMode, }; pub use api::CodexApi; @@ -50,6 +50,7 @@ fn fetch_result( cost: Option, source: &str, account_identity: Option, + reset_credits: Option<&api::ResetCredits>, ) -> ProviderFetchResult { let account_email = usage.account_email.clone(); let mut result = ProviderFetchResult::new(usage, source); @@ -67,6 +68,17 @@ fn fetch_result( if let Some(account_identity) = account_identity { result = result.with_account_identity(account_identity); } + if let Some(credits) = reset_credits { + result = result.with_inventory_item(ProviderInventoryItem { + id: "reset-credits".to_string(), + title: "Reset credits".to_string(), + available_count: credits.available_count, + next_expires_at: api::next_available_reset_credit_expiry( + &credits.credits, + chrono::Utc::now(), + ), + }); + } result } @@ -126,7 +138,7 @@ impl Provider for CodexProvider { let version = detect_codex_version(); match self.api.fetch_usage_pat(version.as_deref()).await { Ok((usage, cost, account_identity)) => { - return Ok(fetch_result(usage, cost, "pat", account_identity)); + return Ok(fetch_result(usage, cost, "pat", account_identity, None)); } Err(error) if pat_allows_auto_fallback(&error) => { tracing::debug!("Codex PAT unavailable in Auto; trying OAuth: {error}"); @@ -135,10 +147,14 @@ impl Provider for CodexProvider { } } - match self.api.fetch_usage().await { - Ok((usage, cost, account_identity)) => { - Ok(fetch_result(usage, cost, "oauth", account_identity)) - } + match self.api.fetch_usage_with_reset_credits().await { + Ok((usage, cost, account_identity, reset_credits)) => Ok(fetch_result( + usage, + cost, + "oauth", + account_identity, + reset_credits.as_ref(), + )), Err(error) => { tracing::warn!("Codex API fetch failed: {error}"); Err(error)