From 371d77846abb3a450c5dcfd747c54d2b2839d6bd Mon Sep 17 00:00:00 2001 From: "ccf-lisa[bot]" <286799724+ccf-lisa[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:22:00 -0300 Subject: [PATCH 1/5] feat(api): admin agent instances, CORS and authz coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fifteenth and last layer of the agent remote-configuration stack (split from #465): GET /api/admin/agents/{id}/instances lists an agent's reporting instances with derived status, sync status and per-status counts, and GET …/instances/{instanceId} returns one with its redacted base/effective config; CORS allows If-Match/If-None-Match and exposes ETag for the UI; the server recovers handler panics as 500s; builtin/Cedar authz matrices over every admin agent-config route, and the Cedar docs. The tree now matches #465. Co-Authored-By: Claude Opus 5.5 --- docs/authz-oss-cedar.md | 16 +- docs/docs.go | 375 ++++++++++++++++++ docs/swagger.json | 375 ++++++++++++++++++ docs/swagger.yaml | 255 ++++++++++++ internal/api/handler/agent_config.go | 218 ++++++++++ .../agent_config_admin_integration_test.go | 245 ++++++++++++ internal/api/server.go | 5 +- 7 files changed, 1485 insertions(+), 4 deletions(-) diff --git a/docs/authz-oss-cedar.md b/docs/authz-oss-cedar.md index 626c8471..8f5e4ada 100644 --- a/docs/authz-oss-cedar.md +++ b/docs/authz-oss-cedar.md @@ -13,7 +13,7 @@ class). Attribute-rich (ABAC/ReBAC) policy is an Enterprise / bring-your-own-PDP | Driver | What it is | | --------- | ---------------------------------------------------------------------- | -| `builtin` | **Default.** Pre-authz rules: authenticated = allowed, admin via SSO groups. Zero behavior change. | +| `builtin` | **Default.** Pre-authz rules: authenticated = allowed, admin via SSO groups. The `agent` resource (agent reads and remote configuration) also requires the admin check for users; agent service accounts may only register, ingest and sync. | | `cedar` | Embedded Cedar RBAC against the bundled role policies (this document). | | `authzen` | Delegate every decision to a remote AuthZen-compliant PDP (bring your own). | @@ -28,7 +28,7 @@ CCF_AUTHZ_CEDAR_POLICY_DIR=/etc/ccf/policies # optional operator .cedar files ## Bundled roles -Four fixed global roles plus the agent service role, defined in the manifest's `roles:` block +Four fixed global roles plus the agent service role (the manifest lists every role, e.g. `ssp-subscriber`), defined in the manifest's `roles:` block (`internal/authz/manifest.yaml`) and compiled to Cedar policies at startup: | Role | Grants | @@ -37,7 +37,17 @@ Four fixed global roles plus the agent service role, defined in the manifest's ` | `contributor` | Author content (OSCAL docs, risk/poam register, workflows, dashboards, evidence); read everything; no admin. | | `auditor` | Read everything; record evidence; maintain the risk/poam register. | | `viewer` | Read everything; no writes. | -| `agent` | Service accounts: ingest evidence/heartbeats, register. | +| `agent` | Service accounts: ingest evidence/heartbeats, register, sync their remote configuration. | + +viewer, auditor and contributor read agent configurations through `"*": [read]`. This is +intended; narrow it with an operator `forbid` policy if needed. The instance reports +(`base`/`effective`) are redacted. `GET …/config` and `GET …/config/revisions/{rev}` return +the **overlay** verbatim only to callers that also hold `agent:configure` (editors), so it +can be edited; every other `agent:read` holder gets it redacted with the report rules +(secret-like keys and values become `••••`). If the configure check cannot be evaluated, +the overlay is redacted. Still, prefer `${env:NAME}` placeholders over literal secrets in +an overlay: editors and every stored revision keep the literal. Deleting an agent deletes +its revisions. Previewing an overlay (`POST …/config/preview`) needs `agent:configure`. Cedar is **deny-by-default**: a subject with no assigned role is denied every request. diff --git a/docs/docs.go b/docs/docs.go index b8d201c2..ff3399c2 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -526,6 +526,129 @@ const docTemplate = `{ ] } }, + "/admin/agents/{id}/instances": { + "get": { + "description": "Summaries of the instances that reported or heartbeated with a config digest, with the derived status (pending and unknown are server-derived), sync status, staleness and counts. Base/effective configs are on the instance detail route.", + "produces": [ + "application/json" + ], + "tags": [ + "Agent Configuration" + ], + "summary": "List an agent's instances", + "parameters": [ + { + "type": "string", + "description": "Agent ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/handler.agentInstanceListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.Error" + } + } + }, + "security": [ + { + "OAuth2Password": [] + } + ] + } + }, + "/admin/agents/{id}/instances/{instanceId}": { + "get": { + "description": "The instance's summary plus its redacted base and effective configs (snake_case). instanceId is the agent-side instance UUID.", + "produces": [ + "application/json" + ], + "tags": [ + "Agent Configuration" + ], + "summary": "Get one agent instance", + "parameters": [ + { + "type": "string", + "description": "Agent ID", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Instance ID", + "name": "instanceId", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/handler.GenericDataResponse-handler_agentInstanceDetail" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.Error" + } + } + }, + "security": [ + { + "OAuth2Password": [] + } + ] + } + }, "/admin/ai-diagnostics/runs": { "get": { "description": "Lists dashboard suggestion runs across all SSPs, newest first, with optional status and SSP filters.", @@ -37365,6 +37488,19 @@ const docTemplate = `{ } } }, + "handler.GenericDataResponse-handler_agentInstanceDetail": { + "type": "object", + "properties": { + "data": { + "description": "Wrapped response data", + "allOf": [ + { + "$ref": "#/definitions/handler.agentInstanceDetail" + } + ] + } + } + }, "handler.GenericDataResponse-handler_bulkControlLinkResponse": { "type": "object", "properties": { @@ -39429,6 +39565,245 @@ const docTemplate = `{ } } }, + "handler.agentInstanceCounts": { + "type": "object", + "properties": { + "failed": { + "type": "integer" + }, + "fresh": { + "type": "integer" + }, + "in-sync": { + "type": "integer" + }, + "out-of-sync": { + "type": "integer" + }, + "pending": { + "type": "integer" + }, + "rejected": { + "type": "integer" + }, + "stale": { + "type": "integer" + }, + "total": { + "type": "integer" + }, + "unknown": { + "type": "integer" + } + } + }, + "handler.agentInstanceDetail": { + "type": "object", + "properties": { + "agent-version": { + "type": "string" + }, + "applied-revision": { + "type": "integer" + }, + "attempted-revision": { + "type": "integer" + }, + "base": { + "type": "object" + }, + "daemon": { + "type": "boolean" + }, + "effective": { + "type": "object" + }, + "effective-digest": { + "type": "string" + }, + "error": { + "type": "string" + }, + "first-seen-at": { + "type": "string" + }, + "heartbeat-config-revision": { + "type": "integer" + }, + "hostname": { + "type": "string" + }, + "instance-id": { + "type": "string" + }, + "last-seen-at": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "plugins": { + "description": "Plugins are the reported plugins and the agent library each was built with (R76).\nEmpty until an agent that reports them does.", + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.PluginReport" + } + }, + "reason": { + "type": "string" + }, + "remote-config": { + "type": "object" + }, + "report-stale": { + "description": "heartbeat digest != reported digest", + "type": "boolean" + }, + "reported-at": { + "type": "string" + }, + "stale": { + "type": "boolean" + }, + "status": { + "description": "applied|rejected|failed|not-applicable|pending|unknown", + "type": "string" + }, + "sync-status": { + "description": "in-sync|out-of-sync|not-applicable|unknown", + "type": "string" + }, + "truncated": { + "type": "boolean" + }, + "unsafe": { + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.Change" + } + }, + "warnings": { + "description": "R41", + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.FieldError" + } + } + } + }, + "handler.agentInstanceListResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/handler.agentInstanceSummary" + } + }, + "meta": { + "$ref": "#/definitions/handler.agentInstancesMeta" + } + } + }, + "handler.agentInstanceSummary": { + "type": "object", + "properties": { + "agent-version": { + "type": "string" + }, + "applied-revision": { + "type": "integer" + }, + "attempted-revision": { + "type": "integer" + }, + "daemon": { + "type": "boolean" + }, + "effective-digest": { + "type": "string" + }, + "error": { + "type": "string" + }, + "first-seen-at": { + "type": "string" + }, + "heartbeat-config-revision": { + "type": "integer" + }, + "hostname": { + "type": "string" + }, + "instance-id": { + "type": "string" + }, + "last-seen-at": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "plugins": { + "description": "Plugins are the reported plugins and the agent library each was built with (R76).\nEmpty until an agent that reports them does.", + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.PluginReport" + } + }, + "reason": { + "type": "string" + }, + "remote-config": { + "type": "object" + }, + "report-stale": { + "description": "heartbeat digest != reported digest", + "type": "boolean" + }, + "reported-at": { + "type": "string" + }, + "stale": { + "type": "boolean" + }, + "status": { + "description": "applied|rejected|failed|not-applicable|pending|unknown", + "type": "string" + }, + "sync-status": { + "description": "in-sync|out-of-sync|not-applicable|unknown", + "type": "string" + }, + "truncated": { + "type": "boolean" + }, + "unsafe": { + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.Change" + } + }, + "warnings": { + "description": "R41", + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.FieldError" + } + } + } + }, + "handler.agentInstancesMeta": { + "type": "object", + "properties": { + "counts": { + "$ref": "#/definitions/handler.agentInstanceCounts" + }, + "desired-revision": { + "type": "integer" + } + } + }, "handler.attachFilterResponsibilityRequest": { "type": "object", "required": [ diff --git a/docs/swagger.json b/docs/swagger.json index d791b2ba..b4417150 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -520,6 +520,129 @@ ] } }, + "/admin/agents/{id}/instances": { + "get": { + "description": "Summaries of the instances that reported or heartbeated with a config digest, with the derived status (pending and unknown are server-derived), sync status, staleness and counts. Base/effective configs are on the instance detail route.", + "produces": [ + "application/json" + ], + "tags": [ + "Agent Configuration" + ], + "summary": "List an agent's instances", + "parameters": [ + { + "type": "string", + "description": "Agent ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/handler.agentInstanceListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.Error" + } + } + }, + "security": [ + { + "OAuth2Password": [] + } + ] + } + }, + "/admin/agents/{id}/instances/{instanceId}": { + "get": { + "description": "The instance's summary plus its redacted base and effective configs (snake_case). instanceId is the agent-side instance UUID.", + "produces": [ + "application/json" + ], + "tags": [ + "Agent Configuration" + ], + "summary": "Get one agent instance", + "parameters": [ + { + "type": "string", + "description": "Agent ID", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Instance ID", + "name": "instanceId", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/handler.GenericDataResponse-handler_agentInstanceDetail" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.Error" + } + } + }, + "security": [ + { + "OAuth2Password": [] + } + ] + } + }, "/admin/ai-diagnostics/runs": { "get": { "description": "Lists dashboard suggestion runs across all SSPs, newest first, with optional status and SSP filters.", @@ -37359,6 +37482,19 @@ } } }, + "handler.GenericDataResponse-handler_agentInstanceDetail": { + "type": "object", + "properties": { + "data": { + "description": "Wrapped response data", + "allOf": [ + { + "$ref": "#/definitions/handler.agentInstanceDetail" + } + ] + } + } + }, "handler.GenericDataResponse-handler_bulkControlLinkResponse": { "type": "object", "properties": { @@ -39423,6 +39559,245 @@ } } }, + "handler.agentInstanceCounts": { + "type": "object", + "properties": { + "failed": { + "type": "integer" + }, + "fresh": { + "type": "integer" + }, + "in-sync": { + "type": "integer" + }, + "out-of-sync": { + "type": "integer" + }, + "pending": { + "type": "integer" + }, + "rejected": { + "type": "integer" + }, + "stale": { + "type": "integer" + }, + "total": { + "type": "integer" + }, + "unknown": { + "type": "integer" + } + } + }, + "handler.agentInstanceDetail": { + "type": "object", + "properties": { + "agent-version": { + "type": "string" + }, + "applied-revision": { + "type": "integer" + }, + "attempted-revision": { + "type": "integer" + }, + "base": { + "type": "object" + }, + "daemon": { + "type": "boolean" + }, + "effective": { + "type": "object" + }, + "effective-digest": { + "type": "string" + }, + "error": { + "type": "string" + }, + "first-seen-at": { + "type": "string" + }, + "heartbeat-config-revision": { + "type": "integer" + }, + "hostname": { + "type": "string" + }, + "instance-id": { + "type": "string" + }, + "last-seen-at": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "plugins": { + "description": "Plugins are the reported plugins and the agent library each was built with (R76).\nEmpty until an agent that reports them does.", + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.PluginReport" + } + }, + "reason": { + "type": "string" + }, + "remote-config": { + "type": "object" + }, + "report-stale": { + "description": "heartbeat digest != reported digest", + "type": "boolean" + }, + "reported-at": { + "type": "string" + }, + "stale": { + "type": "boolean" + }, + "status": { + "description": "applied|rejected|failed|not-applicable|pending|unknown", + "type": "string" + }, + "sync-status": { + "description": "in-sync|out-of-sync|not-applicable|unknown", + "type": "string" + }, + "truncated": { + "type": "boolean" + }, + "unsafe": { + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.Change" + } + }, + "warnings": { + "description": "R41", + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.FieldError" + } + } + } + }, + "handler.agentInstanceListResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/handler.agentInstanceSummary" + } + }, + "meta": { + "$ref": "#/definitions/handler.agentInstancesMeta" + } + } + }, + "handler.agentInstanceSummary": { + "type": "object", + "properties": { + "agent-version": { + "type": "string" + }, + "applied-revision": { + "type": "integer" + }, + "attempted-revision": { + "type": "integer" + }, + "daemon": { + "type": "boolean" + }, + "effective-digest": { + "type": "string" + }, + "error": { + "type": "string" + }, + "first-seen-at": { + "type": "string" + }, + "heartbeat-config-revision": { + "type": "integer" + }, + "hostname": { + "type": "string" + }, + "instance-id": { + "type": "string" + }, + "last-seen-at": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "plugins": { + "description": "Plugins are the reported plugins and the agent library each was built with (R76).\nEmpty until an agent that reports them does.", + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.PluginReport" + } + }, + "reason": { + "type": "string" + }, + "remote-config": { + "type": "object" + }, + "report-stale": { + "description": "heartbeat digest != reported digest", + "type": "boolean" + }, + "reported-at": { + "type": "string" + }, + "stale": { + "type": "boolean" + }, + "status": { + "description": "applied|rejected|failed|not-applicable|pending|unknown", + "type": "string" + }, + "sync-status": { + "description": "in-sync|out-of-sync|not-applicable|unknown", + "type": "string" + }, + "truncated": { + "type": "boolean" + }, + "unsafe": { + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.Change" + } + }, + "warnings": { + "description": "R41", + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.FieldError" + } + } + } + }, + "handler.agentInstancesMeta": { + "type": "object", + "properties": { + "counts": { + "$ref": "#/definitions/handler.agentInstanceCounts" + }, + "desired-revision": { + "type": "integer" + } + } + }, "handler.attachFilterResponsibilityRequest": { "type": "object", "required": [ diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 931557e9..288e19f1 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1669,6 +1669,13 @@ definitions: - $ref: '#/definitions/handler.agentConfigRevisionResponse' description: Wrapped response data type: object + handler.GenericDataResponse-handler_agentInstanceDetail: + properties: + data: + allOf: + - $ref: '#/definitions/handler.agentInstanceDetail' + description: Wrapped response data + type: object handler.GenericDataResponse-handler_bulkControlLinkResponse: properties: data: @@ -2889,6 +2896,171 @@ definitions: revision: type: integer type: object + handler.agentInstanceCounts: + properties: + failed: + type: integer + fresh: + type: integer + in-sync: + type: integer + out-of-sync: + type: integer + pending: + type: integer + rejected: + type: integer + stale: + type: integer + total: + type: integer + unknown: + type: integer + type: object + handler.agentInstanceDetail: + properties: + agent-version: + type: string + applied-revision: + type: integer + attempted-revision: + type: integer + base: + type: object + daemon: + type: boolean + effective: + type: object + effective-digest: + type: string + error: + type: string + first-seen-at: + type: string + heartbeat-config-revision: + type: integer + hostname: + type: string + instance-id: + type: string + last-seen-at: + type: string + mode: + type: string + plugins: + description: |- + Plugins are the reported plugins and the agent library each was built with (R76). + Empty until an agent that reports them does. + items: + $ref: '#/definitions/agentconfig.PluginReport' + type: array + reason: + type: string + remote-config: + type: object + report-stale: + description: heartbeat digest != reported digest + type: boolean + reported-at: + type: string + stale: + type: boolean + status: + description: applied|rejected|failed|not-applicable|pending|unknown + type: string + sync-status: + description: in-sync|out-of-sync|not-applicable|unknown + type: string + truncated: + type: boolean + unsafe: + items: + $ref: '#/definitions/agentconfig.Change' + type: array + warnings: + description: R41 + items: + $ref: '#/definitions/agentconfig.FieldError' + type: array + type: object + handler.agentInstanceListResponse: + properties: + data: + items: + $ref: '#/definitions/handler.agentInstanceSummary' + type: array + meta: + $ref: '#/definitions/handler.agentInstancesMeta' + type: object + handler.agentInstanceSummary: + properties: + agent-version: + type: string + applied-revision: + type: integer + attempted-revision: + type: integer + daemon: + type: boolean + effective-digest: + type: string + error: + type: string + first-seen-at: + type: string + heartbeat-config-revision: + type: integer + hostname: + type: string + instance-id: + type: string + last-seen-at: + type: string + mode: + type: string + plugins: + description: |- + Plugins are the reported plugins and the agent library each was built with (R76). + Empty until an agent that reports them does. + items: + $ref: '#/definitions/agentconfig.PluginReport' + type: array + reason: + type: string + remote-config: + type: object + report-stale: + description: heartbeat digest != reported digest + type: boolean + reported-at: + type: string + stale: + type: boolean + status: + description: applied|rejected|failed|not-applicable|pending|unknown + type: string + sync-status: + description: in-sync|out-of-sync|not-applicable|unknown + type: string + truncated: + type: boolean + unsafe: + items: + $ref: '#/definitions/agentconfig.Change' + type: array + warnings: + description: R41 + items: + $ref: '#/definitions/agentconfig.FieldError' + type: array + type: object + handler.agentInstancesMeta: + properties: + counts: + $ref: '#/definitions/handler.agentInstanceCounts' + desired-revision: + type: integer + type: object handler.attachFilterResponsibilityRequest: properties: controlId: @@ -13079,6 +13251,89 @@ paths: summary: Revert an agent's configuration to an earlier revision tags: - Agent Configuration + /admin/agents/{id}/instances: + get: + description: Summaries of the instances that reported or heartbeated with a + config digest, with the derived status (pending and unknown are server-derived), + sync status, staleness and counts. Base/effective configs are on the instance + detail route. + parameters: + - description: Agent ID + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/handler.agentInstanceListResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/api.Error' + "403": + description: Forbidden + schema: + $ref: '#/definitions/api.Error' + "404": + description: Not Found + schema: + $ref: '#/definitions/api.Error' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/api.Error' + security: + - OAuth2Password: [] + summary: List an agent's instances + tags: + - Agent Configuration + /admin/agents/{id}/instances/{instanceId}: + get: + description: The instance's summary plus its redacted base and effective configs + (snake_case). instanceId is the agent-side instance UUID. + parameters: + - description: Agent ID + in: path + name: id + required: true + type: string + - description: Instance ID + in: path + name: instanceId + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/handler.GenericDataResponse-handler_agentInstanceDetail' + "400": + description: Bad Request + schema: + $ref: '#/definitions/api.Error' + "403": + description: Forbidden + schema: + $ref: '#/definitions/api.Error' + "404": + description: Not Found + schema: + $ref: '#/definitions/api.Error' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/api.Error' + security: + - OAuth2Password: [] + summary: Get one agent instance + tags: + - Agent Configuration /admin/ai-diagnostics/runs: get: description: Lists dashboard suggestion runs across all SSPs, newest first, diff --git a/internal/api/handler/agent_config.go b/internal/api/handler/agent_config.go index 024dd748..7cb1b111 100644 --- a/internal/api/handler/agent_config.go +++ b/internal/api/handler/agent_config.go @@ -61,6 +61,8 @@ func (h *AgentConfigHandler) Register(g *echo.Group, guard middleware.ResourceGu g.GET("/:id/config/revisions", h.ListRevisions, guard.Read()) g.GET("/:id/config/revisions/:rev", h.GetRevision, guard.Read()) g.POST("/:id/config/revisions/:rev/revert", h.Revert, write) + g.GET("/:id/instances", h.ListInstances, guard.Read()) + g.GET("/:id/instances/:instanceId", h.GetInstance, guard.Read()) } // ---- DTOs (A4.4) ---- @@ -76,6 +78,62 @@ type agentConfigRevisionResponse struct { RevertOf *int64 `json:"revert-of"` } +type agentInstanceSummary struct { + InstanceID string `json:"instance-id"` + Hostname *string `json:"hostname"` + AgentVersion *string `json:"agent-version"` + Mode string `json:"mode"` + Daemon *bool `json:"daemon"` + FirstSeenAt time.Time `json:"first-seen-at"` + LastSeenAt time.Time `json:"last-seen-at"` + ReportedAt *time.Time `json:"reported-at"` + Stale bool `json:"stale"` + AppliedRevision *int64 `json:"applied-revision"` + AttemptedRevision *int64 `json:"attempted-revision"` + Status string `json:"status"` // applied|rejected|failed|not-applicable|pending|unknown + Reason *string `json:"reason"` + Error *string `json:"error"` + Truncated bool `json:"truncated"` + SyncStatus string `json:"sync-status"` // in-sync|out-of-sync|not-applicable|unknown + EffectiveDigest *string `json:"effective-digest"` + HeartbeatConfigRevision *int64 `json:"heartbeat-config-revision"` + ReportStale bool `json:"report-stale"` // heartbeat digest != reported digest + RemoteConfig json.RawMessage `json:"remote-config,omitempty" swaggertype:"object"` + Unsafe []agentconfig.Change `json:"unsafe"` + Warnings []agentconfig.FieldError `json:"warnings"` // R41 + // Plugins are the reported plugins and the agent library each was built with (R76). + // Empty until an agent that reports them does. + Plugins []agentconfig.PluginReport `json:"plugins"` +} + +type agentInstanceDetail struct { + agentInstanceSummary + Base json.RawMessage `json:"base" swaggertype:"object"` + Effective json.RawMessage `json:"effective" swaggertype:"object"` +} + +type agentInstanceCounts struct { + Total int `json:"total"` + Fresh int `json:"fresh"` + Stale int `json:"stale"` + InSync int `json:"in-sync"` + OutOfSync int `json:"out-of-sync"` + Pending int `json:"pending"` + Rejected int `json:"rejected"` + Failed int `json:"failed"` + Unknown int `json:"unknown"` +} + +type agentInstancesMeta struct { + DesiredRevision int64 `json:"desired-revision"` + Counts agentInstanceCounts `json:"counts"` +} + +type agentInstanceListResponse struct { + Data []agentInstanceSummary `json:"data"` + Meta agentInstancesMeta `json:"meta"` +} + type configPreviewResponse struct { DesiredRevision int64 `json:"desired-revision"` Standalone bool `json:"standalone"` @@ -633,8 +691,161 @@ func (h *AgentConfigHandler) GetRevision(ctx echo.Context) error { return ctx.JSON(http.StatusOK, GenericDataResponse[agentConfigRevisionResponse]{Data: resp}) } +// ListInstances godoc +// +// @Summary List an agent's instances +// @Description Summaries of the instances that reported or heartbeated with a config digest, with the derived status (pending and unknown are server-derived), sync status, staleness and counts. Base/effective configs are on the instance detail route. +// @Tags Agent Configuration +// @Produce json +// @Param id path string true "Agent ID" +// @Success 200 {object} handler.agentInstanceListResponse +// @Failure 400 {object} api.Error +// @Failure 403 {object} api.Error +// @Failure 404 {object} api.Error +// @Failure 500 {object} api.Error +// @Security OAuth2Password +// @Router /admin/agents/{id}/instances [get] +func (h *AgentConfigHandler) ListInstances(ctx echo.Context) error { + agent, errResp := h.resolveAgent(ctx) + if agent == nil { + return errResp + } + reqCtx := ctx.Request().Context() + desired, err := h.svc.CurrentRevisionNumber(reqCtx, *agent.ID) + if err != nil { + return h.internalError(ctx, "load agent configuration", err) + } + instances, err := h.svc.ListInstances(reqCtx, *agent.ID) + if err != nil { + return h.internalError(ctx, "list instances", err) + } + now := h.svc.Now() + resp := agentInstanceListResponse{ + Data: make([]agentInstanceSummary, 0, len(instances)), + Meta: agentInstancesMeta{DesiredRevision: desired}, + } + counts := &resp.Meta.Counts + for _, inst := range instances { + s := h.instanceSummary(inst, desired, now) + resp.Data = append(resp.Data, s) + counts.Total++ + if s.Stale { + counts.Stale++ + } else { + counts.Fresh++ + } + switch s.SyncStatus { + case agentcfg.SyncInSync: + counts.InSync++ + case agentcfg.SyncOutOfSync: + counts.OutOfSync++ + } + switch s.Status { + case agentconfig.StatusPending: + counts.Pending++ + case agentconfig.StatusRejected: + counts.Rejected++ + case agentconfig.StatusFailed: + counts.Failed++ + case agentconfig.StatusUnknown: + counts.Unknown++ + } + } + return ctx.JSON(http.StatusOK, resp) +} + +// GetInstance godoc +// +// @Summary Get one agent instance +// @Description The instance's summary plus its redacted base and effective configs (snake_case). instanceId is the agent-side instance UUID. +// @Tags Agent Configuration +// @Produce json +// @Param id path string true "Agent ID" +// @Param instanceId path string true "Instance ID" +// @Success 200 {object} handler.GenericDataResponse[handler.agentInstanceDetail] +// @Failure 400 {object} api.Error +// @Failure 403 {object} api.Error +// @Failure 404 {object} api.Error +// @Failure 500 {object} api.Error +// @Security OAuth2Password +// @Router /admin/agents/{id}/instances/{instanceId} [get] +func (h *AgentConfigHandler) GetInstance(ctx echo.Context) error { + agent, errResp := h.resolveAgent(ctx) + if agent == nil { + return errResp + } + instanceID, err := uuid.Parse(ctx.Param("instanceId")) + if err != nil { + return ctx.JSON(http.StatusBadRequest, api.InvalidUUID()) + } + reqCtx := ctx.Request().Context() + inst, err := h.svc.GetInstance(reqCtx, *agent.ID, instanceID) + if errors.Is(err, agentcfg.ErrNotFound) { + return ctx.JSON(http.StatusNotFound, api.NotFoundCustomMsg("instance not found")) + } + if err != nil { + return h.internalError(ctx, "load instance", err) + } + desired, err := h.svc.CurrentRevisionNumber(reqCtx, *agent.ID) + if err != nil { + return h.internalError(ctx, "load agent configuration", err) + } + detail := agentInstanceDetail{ + agentInstanceSummary: h.instanceSummary(*inst, desired, h.svc.Now()), + Base: rawOrNull(inst.BaseConfig), + Effective: rawOrNull(inst.EffectiveConfig), + } + return ctx.JSON(http.StatusOK, GenericDataResponse[agentInstanceDetail]{Data: detail}) +} + // ---- helpers ---- +func (h *AgentConfigHandler) instanceSummary(inst relational.AgentInstance, desired int64, now time.Time) agentInstanceSummary { + s := agentInstanceSummary{ + InstanceID: inst.InstanceID.String(), + Hostname: inst.Hostname, + AgentVersion: inst.AgentVersion, + Mode: inst.Mode, + Daemon: inst.Daemon, + FirstSeenAt: inst.FirstSeenAt.UTC(), + LastSeenAt: inst.LastSeenAt.UTC(), + ReportedAt: inst.ReportedAt, + Stale: agentcfg.IsStale(inst, now, h.svc.Settings()), + AppliedRevision: inst.AppliedRevision, + AttemptedRevision: inst.AttemptedRevision, + Status: agentcfg.DeriveStatus(inst, desired), + Reason: inst.ApplyReason, + Error: inst.ApplyError, + Truncated: inst.Truncated, + SyncStatus: agentcfg.DeriveSyncStatus(inst, desired), + EffectiveDigest: inst.EffectiveDigest, + HeartbeatConfigRevision: inst.HeartbeatConfigRevision, + ReportStale: inst.HeartbeatConfigDigest != nil && inst.EffectiveDigest != nil && + *inst.HeartbeatConfigDigest != *inst.EffectiveDigest, + Unsafe: []agentconfig.Change{}, + Warnings: []agentconfig.FieldError{}, + Plugins: []agentconfig.PluginReport{}, + } + if len(inst.RemoteConfig) > 0 && string(inst.RemoteConfig) != "null" { + s.RemoteConfig = json.RawMessage(inst.RemoteConfig) + } + h.decodeColumn(&inst, "unsafe_changes", inst.UnsafeChanges, &s.Unsafe) + h.decodeColumn(&inst, "warnings", inst.Warnings, &s.Warnings) + h.decodeColumn(&inst, "plugins", inst.Plugins, &s.Plugins) + return s +} + +// decodeColumn decodes a stored JSON column into dst, leaving dst untouched when the column +// is empty or does not decode (logged). +func (h *AgentConfigHandler) decodeColumn(inst *relational.AgentInstance, column string, raw []byte, dst any) { + if len(raw) == 0 || string(raw) == "null" { + return + } + if err := json.Unmarshal(raw, dst); err != nil { + h.sugar.Warnw("Failed to decode agent instance column", "instanceID", inst.InstanceID, "column", column, "error", err) + } +} + // resolveAgent loads :id. On failure it returns nil and the already-written error response. func (h *AgentConfigHandler) resolveAgent(ctx echo.Context) (*relational.Agent, error) { id, err := uuid.Parse(ctx.Param("id")) @@ -796,6 +1007,13 @@ func compactJSON(raw json.RawMessage) (json.RawMessage, error) { return buf.Bytes(), nil } +func rawOrNull(raw []byte) json.RawMessage { + if len(raw) == 0 { + return json.RawMessage("null") + } + return json.RawMessage(raw) +} + // nonNil returns an empty (non-nil) slice for nil, so JSON renders [] rather than null. func nonNil[T any](s []T) []T { if s == nil { diff --git a/internal/api/handler/agent_config_admin_integration_test.go b/internal/api/handler/agent_config_admin_integration_test.go index 57db4818..56d9247e 100644 --- a/internal/api/handler/agent_config_admin_integration_test.go +++ b/internal/api/handler/agent_config_admin_integration_test.go @@ -17,8 +17,10 @@ import ( "github.com/compliance-framework/api/internal/api/middleware" "github.com/compliance-framework/api/internal/authn" "github.com/compliance-framework/api/internal/authz" + "github.com/compliance-framework/api/internal/config" "github.com/compliance-framework/api/internal/service/relational" "github.com/compliance-framework/api/internal/service/relational/agentcfg" + "github.com/compliance-framework/api/internal/service/sso" "github.com/compliance-framework/api/internal/tests" "github.com/compliance-framework/api/pkg/agentconfig" "github.com/google/uuid" @@ -211,6 +213,8 @@ func acaData[T any](s *AgentConfigAdminIntegrationSuite, rec *httptest.ResponseR // ---- instance helpers ---- +func acaI64(v int64) *int64 { return &v } + // acaBase returns a reported base (redacted as the agent does: Redact clears the client secret) in the given mode with the vendor ssh plugin // plus any extra plugins. func acaBase(mode string, extra map[string]*agentconfig.Plugin) json.RawMessage { @@ -801,6 +805,132 @@ func (s *AgentConfigAdminIntegrationSuite) TestFileOriginErrorsDoNotBlock() { // ---- Instances ---- +func (s *AgentConfigAdminIntegrationSuite) TestInstances() { + ctx := context.Background() + agentID := *s.agent.ID + + // Desired revision 1. + s.save(`"0"`, `{"verbosity":1}`, 1) + + inSync := s.report(agentID, agentconfig.ModeApplySafe, func(r *agentconfig.Report) { + r.AppliedRevision = acaI64(1) + r.Plugins = []agentconfig.PluginReport{{Name: "ssh", Source: acaVendorPlugin, LibVersion: "v0.7.1"}, {Name: "local"}} + }) + pending := s.report(agentID, agentconfig.ModeApplySafe, nil) // applied nil, attempted nil + rejected := s.report(agentID, agentconfig.ModeApplyAll, func(r *agentconfig.Report) { + r.AttemptedRevision = acaI64(1) + r.Status = agentconfig.StatusRejected + r.Reason = agentconfig.ReasonDownloadFailed + }) + s.makeStale(rejected, time.Hour) + reportOnly := s.report(agentID, agentconfig.ModeReport, nil) + heartbeatOnly := uuid.New() + digest := acaDigest + s.Require().NoError(s.svc.TouchFromHeartbeat(ctx, agentID, nil, heartbeatOnly, acaI64(0), &digest)) + // A newer heartbeat digest than the reported one marks the report stale. + other := acaOtherDigest + s.Require().NoError(s.svc.TouchFromHeartbeat(ctx, agentID, nil, inSync, acaI64(1), &other)) + + // Another agent's instance never shows up. + otherAgent, err := s.CreateAgent("other-agent") + s.Require().NoError(err) + foreign := s.report(*otherAgent.ID, agentconfig.ModeApplySafe, nil) + + rec := s.call(http.MethodGet, s.path("/instances"), nil) + s.Require().Equal(http.StatusOK, rec.Code, rec.Body.String()) + var list agentInstanceListResponse + s.Require().NoError(json.Unmarshal(rec.Body.Bytes(), &list)) + s.Equal(int64(1), list.Meta.DesiredRevision) + s.Equal(agentInstanceCounts{ + Total: 5, Fresh: 4, Stale: 1, + InSync: 1, OutOfSync: 2, + Pending: 1, Rejected: 1, Failed: 0, Unknown: 1, + }, list.Meta.Counts) + + byID := map[string]agentInstanceSummary{} + for _, inst := range list.Data { + byID[inst.InstanceID] = inst + } + s.NotContains(byID, foreign.String()) + + st := byID[inSync.String()] + s.Equal(agentconfig.StatusApplied, st.Status) + s.Equal(agentcfg.SyncInSync, st.SyncStatus) + s.True(st.ReportStale) + s.Equal([]agentconfig.PluginReport{{Name: "ssh", Source: acaVendorPlugin, LibVersion: "v0.7.1"}, {Name: "local"}}, st.Plugins, "R76: listed with the summary") + s.Require().NotNil(st.HeartbeatConfigRevision) + s.Equal(int64(1), *st.HeartbeatConfigRevision) + s.NotEmpty(st.RemoteConfig) + + st = byID[pending.String()] + s.Equal([]agentconfig.PluginReport{}, st.Plugins, "an agent that does not report plugins") + s.Equal(agentconfig.StatusPending, st.Status) + s.Equal(agentcfg.SyncOutOfSync, st.SyncStatus) + s.False(st.Stale) + + st = byID[rejected.String()] + s.Equal(agentconfig.StatusRejected, st.Status) + s.True(st.Stale) + s.Require().NotNil(st.Reason) + s.Equal(agentconfig.ReasonDownloadFailed, *st.Reason) + + st = byID[reportOnly.String()] + s.Equal(agentconfig.StatusNotApplicable, st.Status) + s.Equal(agentcfg.SyncNotApplicable, st.SyncStatus) + + st = byID[heartbeatOnly.String()] + s.Equal(agentconfig.StatusUnknown, st.Status) + s.Equal(agentcfg.SyncUnknown, st.SyncStatus) + s.Nil(st.ReportedAt) + + // Summaries carry [] for list fields and no base/effective. + var rawList struct { + Data []map[string]json.RawMessage `json:"data"` + } + s.Require().NoError(json.Unmarshal(rec.Body.Bytes(), &rawList)) + for _, item := range rawList.Data { + s.NotContains(item, "base") + s.NotContains(item, "effective") + for _, k := range []string{"unsafe", "warnings"} { + s.JSONEq(`[]`, string(item[k]), k) + } + s.NotContains(item, "policy-errors") + s.Contains(item, "plugins") + } + + // Detail: base and effective. + rec = s.call(http.MethodGet, s.path("/instances/"+inSync.String()), nil) + s.Require().Equal(http.StatusOK, rec.Code, rec.Body.String()) + detail := acaData[agentInstanceDetail](s, rec) + s.Equal(inSync.String(), detail.InstanceID) + s.Contains(string(detail.Base), acaVendorPlugin) + s.Contains(string(detail.Effective), acaVendorPolicy) + s.Equal([]agentconfig.PluginReport{{Name: "ssh", Source: acaVendorPlugin, LibVersion: "v0.7.1"}, {Name: "local"}}, detail.Plugins, "R76") + + // A heartbeat-only instance has null configs and [] plugins. + rec = s.call(http.MethodGet, s.path("/instances/"+heartbeatOnly.String()), nil) + s.Require().Equal(http.StatusOK, rec.Code, rec.Body.String()) + var rawDetail struct { + Data map[string]json.RawMessage `json:"data"` + } + s.Require().NoError(json.Unmarshal(rec.Body.Bytes(), &rawDetail)) + s.JSONEq(`null`, string(rawDetail.Data["base"])) + s.NotContains(rawDetail.Data, "policy-bundles") + s.JSONEq(`[]`, string(rawDetail.Data["plugins"])) + + // Another agent's instance => 404; a bad instance id => 400. + rec = s.call(http.MethodGet, s.path("/instances/"+foreign.String()), nil) + s.Equal(http.StatusNotFound, rec.Code, rec.Body.String()) + rec = s.call(http.MethodGet, s.path("/instances/not-a-uuid"), nil) + s.Equal(http.StatusBadRequest, rec.Code, rec.Body.String()) +} + +func (s *AgentConfigAdminIntegrationSuite) TestInstancesEmpty() { + rec := s.call(http.MethodGet, s.path("/instances"), nil) + s.Require().Equal(http.StatusOK, rec.Code, rec.Body.String()) + s.JSONEq(`{"data":[],"meta":{"desired-revision":0,"counts":{"total":0,"fresh":0,"stale":0,"in-sync":0,"out-of-sync":0,"pending":0,"rejected":0,"failed":0,"unknown":0}}}`, rec.Body.String()) +} + // ---- Agent deletion ---- // Deleting an agent removes its instances and its revisions (the purge path for an overlay @@ -831,8 +961,100 @@ func (s *AgentConfigAdminIntegrationSuite) TestDeleteAgentRemovesInstancesAndRev // ---- Builtin authz (R39) ---- +func (s *AgentConfigAdminIntegrationSuite) TestBuiltinSSOUserNeedsAdminGroup() { + original := s.Config.SSO + s.Config.SSO = &config.SSOConfig{ + Enabled: true, + Providers: map[string]config.SSOProviderConfig{ + "test": {Name: "test", RequiredAdminGroups: []string{"ccf-admins"}}, + }, + } + defer func() { s.Config.SSO = original }() + + ssoToken := func(email string, groups []string) string { + user, token := s.userToken(email, "sso", "") + s.Require().NoError(s.DB.Create(&relational.SSOUserLink{ + UserID: user.ID.String(), + Provider: "test", + ExternalID: email, + Email: email, + Groups: sso.SerializeStringArray(groups), + LastSync: time.Now(), + }).Error) + return token + } + member := ssoToken("sso-member@example.com", []string{"developers"}) + admin := ssoToken("sso-admin@example.com", []string{"ccf-admins"}) + + denied := []struct { + method, path string + body []byte + headers []string + }{ + {http.MethodGet, "/api/admin/agents", nil, nil}, + {http.MethodGet, s.path(""), nil, nil}, + {http.MethodGet, s.path("/config"), nil, nil}, + {http.MethodPut, s.path("/config"), acaPutBody(`{"verbosity":1}`), []string{"If-Match", `"0"`}}, + {http.MethodPost, s.path("/config/preview"), acaPutBody(`{"verbosity":1}`), nil}, + {http.MethodGet, s.path("/config/revisions"), nil, nil}, + {http.MethodGet, s.path("/instances"), nil, nil}, + } + for _, tc := range denied { + rec := s.send(s.server, member, tc.method, tc.path, tc.body, tc.headers...) + s.Equal(http.StatusForbidden, rec.Code, "%s %s: %s", tc.method, tc.path, rec.Body.String()) + } + + // An SSO user in the admin group and the password user are allowed. + for _, token := range []string{admin, s.token} { + rec := s.send(s.server, token, http.MethodGet, "/api/admin/agents", nil) + s.Equal(http.StatusOK, rec.Code, rec.Body.String()) + rec = s.send(s.server, token, http.MethodGet, s.path("/config"), nil) + s.Equal(http.StatusOK, rec.Code, rec.Body.String()) + } + rec := s.send(s.server, admin, http.MethodPut, s.path("/config"), acaPutBody(`{"verbosity":1}`), "If-Match", `"0"`) + s.Equal(http.StatusCreated, rec.Code, rec.Body.String()) + s.Equal(int64(1), s.revisionCount(*s.agent.ID)) +} + // ---- Cedar authz (R40) ---- +func (s *AgentConfigAdminIntegrationSuite) TestCedarViewer() { + _, viewer := s.userToken("viewer@example.com", "", "viewer") + srv := s.cedarServer() + + allowed := []struct{ method, path string }{ + {http.MethodGet, "/api/admin/agents"}, + {http.MethodGet, s.path("")}, + {http.MethodGet, s.path("/config")}, + {http.MethodGet, s.path("/config/revisions")}, + {http.MethodGet, s.path("/instances")}, + } + for _, tc := range allowed { + rec := s.send(srv, viewer, tc.method, tc.path, nil) + s.Equal(http.StatusOK, rec.Code, "%s %s: %s", tc.method, tc.path, rec.Body.String()) + } + + denied := []struct { + method, path string + body []byte + headers []string + }{ + {http.MethodPost, s.path("/config/preview"), acaPutBody(`{"verbosity":1}`), nil}, + {http.MethodPut, s.path("/config"), acaPutBody(`{"verbosity":1}`), []string{"If-Match", `"0"`}}, + {http.MethodPost, s.path("/config/revisions/1/revert"), nil, []string{"If-Match", `"0"`}}, + {http.MethodPost, "/api/admin/agents", []byte(`{"name":"viewer-agent"}`), nil}, + {http.MethodPut, s.path(""), []byte(`{"name":"renamed"}`), nil}, + {http.MethodDelete, s.path(""), nil, nil}, + {http.MethodGet, s.path("/keys"), nil, nil}, + {http.MethodPost, s.path("/keys"), []byte(`{"never-expires":true}`), nil}, + } + for _, tc := range denied { + rec := s.send(srv, viewer, tc.method, tc.path, tc.body, tc.headers...) + s.Equal(http.StatusForbidden, rec.Code, "%s %s: %s", tc.method, tc.path, rec.Body.String()) + } + s.Equal(int64(0), s.revisionCount(*s.agent.ID)) +} + // Overlays are verbatim for agent:configure holders and redacted for read-only callers. func (s *AgentConfigAdminIntegrationSuite) TestCedarOverlayRedactedForReaders() { overlay := `{"plugins":{"ssh":{"config":{"password":"hunter2","host":"db","pass_ref":"${env:SSH_PASS}"},"policy_data":{"api_token":"t0k3n","threshold":3}}}}` @@ -911,3 +1133,26 @@ func (s *AgentConfigAdminIntegrationSuite) TestCedarUserWithoutRoleDenied() { } // ---- CORS (R13) ---- + +func (s *AgentConfigAdminIntegrationSuite) TestCORSAllowsIfMatchAndExposesETag() { + const origin = "http://ui.example.com" + original := s.Config.APIAllowedOrigins + s.Config.APIAllowedOrigins = []string{origin} + defer func() { s.Config.APIAllowedOrigins = original }() + srv := s.newServer(nil) + + rec := s.send(srv, "", http.MethodOptions, s.path("/config"), nil, + echo.HeaderOrigin, origin, + echo.HeaderAccessControlRequestMethod, http.MethodPut, + echo.HeaderAccessControlRequestHeaders, "if-match", + ) + s.Require().Equal(http.StatusNoContent, rec.Code, rec.Body.String()) + s.Equal(origin, rec.Header().Get(echo.HeaderAccessControlAllowOrigin)) + s.Contains(strings.ToLower(rec.Header().Get(echo.HeaderAccessControlAllowHeaders)), "if-match") + + rec = s.send(srv, s.token, http.MethodGet, s.path("/config"), nil, echo.HeaderOrigin, origin) + s.Require().Equal(http.StatusOK, rec.Code, rec.Body.String()) + s.Equal(origin, rec.Header().Get(echo.HeaderAccessControlAllowOrigin)) + s.Contains(rec.Header().Get(echo.HeaderAccessControlExposeHeaders), "ETag") + s.Equal(`"0"`, rec.Header().Get("ETag")) +} diff --git a/internal/api/server.go b/internal/api/server.go index 82b6d27b..267afc52 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -48,9 +48,12 @@ func NewServer(ctx context.Context, s *zap.SugaredLogger, config *config.Config, return nil }, })) + // A handler panic becomes a 500 (logged above) instead of a dropped connection. + e.Use(middleware.Recover()) e.Use(middleware.CORSWithConfig(middleware.CORSConfig{ AllowOrigins: config.APIAllowedOrigins, - AllowHeaders: []string{echo.HeaderOrigin, echo.HeaderContentType, echo.HeaderAccept, echo.HeaderAuthorization}, + AllowHeaders: []string{echo.HeaderOrigin, echo.HeaderContentType, echo.HeaderAccept, echo.HeaderAuthorization, "If-Match", "If-None-Match"}, + ExposeHeaders: []string{"ETag"}, AllowCredentials: true, })) e.Use(echoprometheus.NewMiddlewareWithConfig(echoprometheus.MiddlewareConfig{ From 6e7ad5ee8fa2515f9504a45240ece2576fbe0cfe Mon Sep 17 00:00:00 2001 From: "ccf-lisa[bot]" <286799724+ccf-lisa[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 07:37:03 -0300 Subject: [PATCH 2/5] fix(api): return the agent instance list as GenericDataListResponse --- docs/docs.go | 41 ++++++++++++------- docs/swagger.json | 41 ++++++++++++------- docs/swagger.yaml | 25 ++++++----- internal/api/handler/agent_config.go | 19 +++------ .../agent_config_admin_integration_test.go | 5 ++- 5 files changed, 77 insertions(+), 54 deletions(-) diff --git a/docs/docs.go b/docs/docs.go index ff3399c2..21a67517 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -549,7 +549,19 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/handler.agentInstanceListResponse" + "allOf": [ + { + "$ref": "#/definitions/handler.GenericDataListResponse-handler_agentInstanceSummary" + }, + { + "type": "object", + "properties": { + "meta": { + "$ref": "#/definitions/handler.agentInstancesMeta" + } + } + } + ] } }, "400": { @@ -36361,6 +36373,19 @@ const docTemplate = `{ "meta": {} } }, + "handler.GenericDataListResponse-handler_agentInstanceSummary": { + "type": "object", + "properties": { + "data": { + "description": "Items from the list response", + "type": "array", + "items": { + "$ref": "#/definitions/handler.agentInstanceSummary" + } + }, + "meta": {} + } + }, "handler.GenericDataListResponse-handler_availableNotificationProviderResponse": { "type": "object", "properties": { @@ -39691,20 +39716,6 @@ const docTemplate = `{ } } }, - "handler.agentInstanceListResponse": { - "type": "object", - "properties": { - "data": { - "type": "array", - "items": { - "$ref": "#/definitions/handler.agentInstanceSummary" - } - }, - "meta": { - "$ref": "#/definitions/handler.agentInstancesMeta" - } - } - }, "handler.agentInstanceSummary": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index b4417150..2b5624e1 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -543,7 +543,19 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/handler.agentInstanceListResponse" + "allOf": [ + { + "$ref": "#/definitions/handler.GenericDataListResponse-handler_agentInstanceSummary" + }, + { + "type": "object", + "properties": { + "meta": { + "$ref": "#/definitions/handler.agentInstancesMeta" + } + } + } + ] } }, "400": { @@ -36355,6 +36367,19 @@ "meta": {} } }, + "handler.GenericDataListResponse-handler_agentInstanceSummary": { + "type": "object", + "properties": { + "data": { + "description": "Items from the list response", + "type": "array", + "items": { + "$ref": "#/definitions/handler.agentInstanceSummary" + } + }, + "meta": {} + } + }, "handler.GenericDataListResponse-handler_availableNotificationProviderResponse": { "type": "object", "properties": { @@ -39685,20 +39710,6 @@ } } }, - "handler.agentInstanceListResponse": { - "type": "object", - "properties": { - "data": { - "type": "array", - "items": { - "$ref": "#/definitions/handler.agentInstanceSummary" - } - }, - "meta": { - "$ref": "#/definitions/handler.agentInstancesMeta" - } - } - }, "handler.agentInstanceSummary": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 288e19f1..431330ed 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -910,6 +910,15 @@ definitions: type: array meta: {} type: object + handler.GenericDataListResponse-handler_agentInstanceSummary: + properties: + data: + description: Items from the list response + items: + $ref: '#/definitions/handler.agentInstanceSummary' + type: array + meta: {} + type: object handler.GenericDataListResponse-handler_availableNotificationProviderResponse: properties: data: @@ -2983,15 +2992,6 @@ definitions: $ref: '#/definitions/agentconfig.FieldError' type: array type: object - handler.agentInstanceListResponse: - properties: - data: - items: - $ref: '#/definitions/handler.agentInstanceSummary' - type: array - meta: - $ref: '#/definitions/handler.agentInstancesMeta' - type: object handler.agentInstanceSummary: properties: agent-version: @@ -13269,7 +13269,12 @@ paths: "200": description: OK schema: - $ref: '#/definitions/handler.agentInstanceListResponse' + allOf: + - $ref: '#/definitions/handler.GenericDataListResponse-handler_agentInstanceSummary' + - properties: + meta: + $ref: '#/definitions/handler.agentInstancesMeta' + type: object "400": description: Bad Request schema: diff --git a/internal/api/handler/agent_config.go b/internal/api/handler/agent_config.go index 7cb1b111..aef8f951 100644 --- a/internal/api/handler/agent_config.go +++ b/internal/api/handler/agent_config.go @@ -129,11 +129,6 @@ type agentInstancesMeta struct { Counts agentInstanceCounts `json:"counts"` } -type agentInstanceListResponse struct { - Data []agentInstanceSummary `json:"data"` - Meta agentInstancesMeta `json:"meta"` -} - type configPreviewResponse struct { DesiredRevision int64 `json:"desired-revision"` Standalone bool `json:"standalone"` @@ -698,7 +693,7 @@ func (h *AgentConfigHandler) GetRevision(ctx echo.Context) error { // @Tags Agent Configuration // @Produce json // @Param id path string true "Agent ID" -// @Success 200 {object} handler.agentInstanceListResponse +// @Success 200 {object} handler.GenericDataListResponse[handler.agentInstanceSummary]{meta=handler.agentInstancesMeta} // @Failure 400 {object} api.Error // @Failure 403 {object} api.Error // @Failure 404 {object} api.Error @@ -720,14 +715,12 @@ func (h *AgentConfigHandler) ListInstances(ctx echo.Context) error { return h.internalError(ctx, "list instances", err) } now := h.svc.Now() - resp := agentInstanceListResponse{ - Data: make([]agentInstanceSummary, 0, len(instances)), - Meta: agentInstancesMeta{DesiredRevision: desired}, - } - counts := &resp.Meta.Counts + data := make([]agentInstanceSummary, 0, len(instances)) + meta := agentInstancesMeta{DesiredRevision: desired} + counts := &meta.Counts for _, inst := range instances { s := h.instanceSummary(inst, desired, now) - resp.Data = append(resp.Data, s) + data = append(data, s) counts.Total++ if s.Stale { counts.Stale++ @@ -751,7 +744,7 @@ func (h *AgentConfigHandler) ListInstances(ctx echo.Context) error { counts.Unknown++ } } - return ctx.JSON(http.StatusOK, resp) + return ctx.JSON(http.StatusOK, GenericDataListResponse[agentInstanceSummary]{Data: data, Meta: meta}) } // GetInstance godoc diff --git a/internal/api/handler/agent_config_admin_integration_test.go b/internal/api/handler/agent_config_admin_integration_test.go index 56d9247e..b789cac9 100644 --- a/internal/api/handler/agent_config_admin_integration_test.go +++ b/internal/api/handler/agent_config_admin_integration_test.go @@ -838,7 +838,10 @@ func (s *AgentConfigAdminIntegrationSuite) TestInstances() { rec := s.call(http.MethodGet, s.path("/instances"), nil) s.Require().Equal(http.StatusOK, rec.Code, rec.Body.String()) - var list agentInstanceListResponse + var list struct { + Data []agentInstanceSummary `json:"data"` + Meta agentInstancesMeta `json:"meta"` + } s.Require().NoError(json.Unmarshal(rec.Body.Bytes(), &list)) s.Equal(int64(1), list.Meta.DesiredRevision) s.Equal(agentInstanceCounts{ From 1679eeb15d47af2277e6a6a9fe42649dd4ceb130 Mon Sep 17 00:00:00 2001 From: "ccf-lisa[bot]" <286799724+ccf-lisa[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 08:43:29 -0300 Subject: [PATCH 3/5] docs: add an operator guide to agent remote configuration docs/agent-remote-config.md covers the agent and admin routes with their status codes, the ETag/If-Match and If-None-Match flows, apply modes and change classification, trusted_sources / overridable_config_flags / allow_local_sources, instances (cap, pruning, report bounds), who sees redacted or verbatim overlays, and the CCF_AGENT_* settings. Co-Authored-By: Claude Opus 5.5 --- docs/agent-remote-config.md | 199 ++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 docs/agent-remote-config.md diff --git a/docs/agent-remote-config.md b/docs/agent-remote-config.md new file mode 100644 index 00000000..d6c90f9c --- /dev/null +++ b/docs/agent-remote-config.md @@ -0,0 +1,199 @@ +# Agent remote configuration + +An admin can change a running agent's configuration from the API instead of editing the +file on its host. The API stores an **overlay** per agent: an RFC 7396 JSON merge patch +(snake_case, the agent's config file format) that the agent merges over its own file: +`effective = MergePatch(file, overlay)`. Omitting a key keeps the file's value; `null` +deletes it so the agent's default applies. An empty overlay (`{}`, revision 0) means the +agent runs from its file. + +Every save appends a numbered **revision**; revisions are never edited or deleted, except +when the agent itself is deleted. Agents poll for the current revision, decide on their +own whether to apply it, and report what they run. The host keeps the last word: its +`remote_config` block decides what an overlay may change, and that block can never be set +remotely. + +The shared model (merge, validation, classification, redaction, digests, ETags) is +`pkg/agentconfig`, which the agent imports too. Design IDs cited in the code (`D3`, `R48`, +`O11`, ...) are defined in `compliance-framework/local-dev`: +`docs/agent-remote-config-design.md` and `docs/agent-remote-config-lld-api.md`. + +## Routes + +Agent-facing routes accept agent JWTs only, never a user token or an anonymous caller (even +with public agent endpoints on), and need `agent:sync`. An agent only reads and reports on +its own configuration. + +| Route | Status codes | +| --- | --- | +| `GET /api/agent/config` | `200` the current overlay `{revision, overlay, created-at}` with an `ETag`; `304` when `If-None-Match` matches; `401`, `403`, `500`. A `404` means the API predates remote configuration. | +| `PUT /api/agent/instances/{instanceId}/config-report` | `204` stored; `400` invalid instance ID or report (for example an unknown mode or status, a malformed digest, a base or effective that is not an object, or a NUL character anywhere); `401`, `403`; `409` instance cap reached (back off); `413` body over 4 MiB; `415` not JSON; `500`. | +| `POST /api/agent/heartbeat` | An authenticated heartbeat with a well-formed `config_digest` (sent when the mode is not `off`) also records `config_revision`/`config_digest` and registers the instance. This is authorized by `heartbeat:ingest`, not `agent:sync`. | + +Admin routes take a user token. Reads need `agent:read`; saving, reverting and preview need +`agent:configure`. With the default `builtin` authz driver both require the admin check. +Every admin route returns `400` for a malformed agent ID, `403` without the permission and +`404` for an unknown agent. + +| Route | Status codes | +| --- | --- | +| `GET /api/admin/agents/{id}/config` | `200` current revision with `ETag: ""`. | +| `PUT /api/admin/agents/{id}/config` | `201` new revision; `200` the overlay is semantically unchanged (nothing is created); `400` malformed body (including data after the JSON object), missing `overlay`, or a comment over 2000 characters or containing a NUL; `409` stale `If-Match`, body has `current-revision`; `413` body over 1 MiB; `415` not JSON; `422` validation errors; `428` `If-Match` missing or not a plain revision number. | +| `POST /api/admin/agents/{id}/config/preview` | `200` validation and a per-instance preview, problems included (it never saves); `400`, `413`, `415`. | +| `GET /api/admin/agents/{id}/config/revisions` | `200` revisions newest first, without overlays; `page` (default 1) and `limit` (default 50, max 100). | +| `GET /api/admin/agents/{id}/config/revisions/{rev}` | `200` one revision; `404` unknown revision. | +| `POST /api/admin/agents/{id}/config/revisions/{rev}/revert` | Saves revision `rev`'s overlay as the next revision (`revert-of` records it). Same `If-Match`, validation and status codes as `PUT`; the body (`{"comment": ...}`) is optional. | +| `GET /api/admin/agents/{id}/instances` | `200` every instance's summary with `meta.desired-revision` and `meta.counts`. Not paginated. | +| `GET /api/admin/agents/{id}/instances/{instanceId}` | `200` the summary plus the redacted `base` and `effective` configs; `404` unknown instance. | + +The CORS configuration allows the `If-Match` and `If-None-Match` request headers and exposes +`ETag`, so a browser client can run both flows below. + +## Saving: ETag and If-Match + +1. `GET …/config` returns the current revision and `ETag: "7"` (the plain revision number, + `"0"` before the first save). +2. `PUT …/config` with `If-Match: "7"` (`W/"7"` and `7` are accepted too) and + `{"overlay": {...}, "comment": "..."}`. +3. If another save happened meanwhile, the answer is `409` with `current-revision`: reload, + reapply the change and retry. Without `If-Match` the answer is `428`. + +A save validates the overlay on its own first (unknown keys, types, locked keys, plugin +names, schedules, `${env:}` placement, at most 256 KiB compact, no NUL). It then merges the +overlay over the reported base of every instance in the **validation set** and validates +the result: + +- every fresh instance (seen within `CCF_AGENT_INSTANCE_STALE_AFTER`) in `apply_safe` or + `apply_all` mode that reported a base; +- if there is none, the single most recently reported apply-mode instance, however old; +- if there is none either, only the overlay-level checks run (`standalone`). + +Report-mode instances are never validated against. Instances that report the same base are +validated once and share the result. Only errors the overlay introduces block a save: an +error already present in `Merge(base, {})` comes from the host's own file and is returned as +a warning. A `422` body lists `overlay` errors and, per instance, `errors` and `warnings`. + +Preview runs the same checks without saving, and shows per instance (fresh and stale) the +redacted effective config, its diff against what the instance runs now, the classified +changes and whether the agent would apply them (`will-apply`, `will-apply-reason`). +`validated` marks the instances a save validates against. Preview covers at most 50 +instances and 16 MiB of reported config (validated instances first, then newest first); +`omitted-instances` counts the rest. A save still validates against the whole set. + +## Polling: ETag and If-None-Match + +The agent sends the `ETag` of the overlay it has as `If-None-Match`. When it still matches +the current revision the answer is `304`, and the API answers it without loading the +overlay. The agent ETag is opaque: send it back verbatim, never build one. It names the +stored revision row, not just its number, so a database reset or a re-created agent never +yields a false `304`. + +## Apply modes and classification + +An agent compares each new revision with its own file and classifies every changed path as +`safe`, `unsafe` or `forbidden`. Its mode (`remote_config.mode` in its file) decides what it +applies: + +| Mode | Applies | +| --- | --- | +| `off` | Nothing (`mode-off`). The default without API credentials. | +| `report` | Nothing (`mode-report`), but reports what it runs. The default with credentials. | +| `apply_safe` | A revision whose changes are all safe; otherwise `unsafe-changes`. | +| `apply_all` | A revision with safe and unsafe changes. | + +In both apply modes a revision with any forbidden change is refused (`forbidden-changes`), +and so is an invalid merged config (`invalid-config`). The classification, with its reason +codes: + +| Change | Class | +| --- | --- | +| `api`, `daemon` or `remote_config` (locked keys; a save rejects them anyway) | forbidden, `locked-key` | +| `verbosity`, `agent_evidence.*` | safe, `logging` | +| A plugin's `schedule`, `labels`, `policy_behavior`, `protocol_version`, `policy_data`, or disabling it | safe, `data-only` | +| Removing a plugin or some of its `policies` | safe, `reduces-scope` | +| A plugin `source` or added `policies` entry already used by an enabled plugin of the file | safe, `already-used` | +| …that is a local path (not an OCI reference) | forbidden, `local-source-not-allowed`; unsafe, `new-local-source` in `apply_all` with `allow_local_sources` | +| …that matches `trusted_sources` | safe, `trusted-source` | +| …any other source | unsafe, `untrusted-source` | +| `plugins.

.config.` matching `overridable_config_flags` | safe, `overridable-config-flag` | +| any other config value | unsafe, `config-not-overridable` | +| A config value with a new `${env:NAME}` reference | unsafe, `new-env-reference`; forbidden for `CCF_API_AUTH_*`, `forbidden-env-reference` | +| Re-enabling a plugin the file disables | unsafe, `reenables-plugin`, or safe when its source is trusted. Its sources and env references are classified as new ones, so a local source is forbidden unless `apply_all` allows local sources. | + +A new plugin takes the class of its parts. So by design `apply_safe` lets an admin change +the policy inputs that decide pass/fail (`policy_data`) or stop a plugin, with no host +opt-in. + +The host's `remote_config` block holds these settings. It is set locally only (file, host +environment or CLI flags), never by an overlay: + +| Setting | Default | Meaning | +| --- | --- | --- | +| `mode` | `report` with credentials, `off` without | See above. | +| `poll_interval` | `60s` | How often the agent polls; at least `15s`. | +| `trusted_sources` | `[]` | `path.Match` globs of plugin and policy sources an overlay may introduce safely, for example `ghcr.io/compliance-framework/*`. Case-sensitive; `*` does not cross `/`. A local path is never trusted. | +| `overridable_config_flags` | `[]` | Plugin config keys an overlay may change safely: `:`, or `` for any plugin, for example `ssh:port`. | +| `allow_local_sources` | `false` | Lets `apply_all` run local-path plugin and policy sources an overlay introduces. | + +Preview classifies with each instance's reported `remote_config` block and its reported +mode, so it shows what each agent would do. + +## Instances + +An instance is one running agent process (agents send an instance UUID). It is registered +by its first config report, or by an authenticated heartbeat that carries a config digest. +The list shows, per instance, the derived `status` (`applied`, `rejected`, `failed` and +`not-applicable` as reported; `pending` when an apply-mode instance has not attempted the +current revision yet; `unknown` before its first report), the `sync-status` (`in-sync`, +`out-of-sync`, `not-applicable` in report or off mode, `unknown`), `stale` (not seen within +`CCF_AGENT_INSTANCE_STALE_AFTER`) and `report-stale` (its heartbeat digest no longer matches +its last report). + +**Cap.** An agent has at most `CCF_AGENT_MAX_INSTANCES` instances that are not yet eligible +for pruning. A new instance at the cap replaces the oldest stale one, since a restarted +agent gets a new instance ID. When every counted instance is fresh, a report from a new +instance gets `409` and its heartbeats are not recorded (logged at most once a minute per +agent). Existing instances keep reporting. + +**Pruning.** A periodic worker job deletes one-shot instances (`daemon: false`) not seen for +`CCF_AGENT_INSTANCE_ONESHOT_RETENTION` and every other instance not seen for +`CCF_AGENT_INSTANCE_RETENTION`. It needs the worker service and runs at most hourly. +Deleting an agent deletes its instances and revisions. + +**Report bounds.** The API caps what a report stores and marks the instance `truncated`: +plugins, warnings and unsafe changes by count, their strings, the error text, hostname and +version by length, and the reported `remote_config` (at most 100 `trusted_sources` and 100 +`overridable_config_flags`, each at most 256 bytes encoded, dropped rather than cut; about +64 KiB in all). The instance list returns these summary columns for every instance, without +the base and effective configs; in the worst case that is about 3 MiB per instance. + +## Who sees secrets + +- **The agent** receives its own overlay verbatim: it has to apply it. +- **Overlays on the admin routes** (`GET …/config`, `GET …/config/revisions/{rev}`) are + verbatim only for callers that also hold `agent:configure`, so they can edit them. Every + other `agent:read` holder gets them redacted (secret-like keys and values become `••••`), + and so does every caller when the `agent:configure` check cannot be evaluated. The + revision list never includes overlays. +- **Reported configs** (`base` and `effective` on the instance detail, and the effective + config in preview) are always redacted. The agent redacts them and the API redacts them + again as a best effort. Report free text that contains a secret (the error, warning + messages, plugin sources, unsafe change values, `remote_config` strings) is replaced with + `••••`. + +Editors and every stored revision keep literal values, so put `${env:NAME}` placeholders in +`plugins.

.config` values instead of secrets: the agent resolves them from its host +environment, and `CCF_API_AUTH_*` can never be referenced. + +## Configuration + +| Variable | Default | Meaning | +| --- | --- | --- | +| `CCF_AGENT_INSTANCE_STALE_AFTER` | `10m` | An instance seen (report or heartbeat) within this window is fresh: saves validate against it, and a full cap cannot replace it. | +| `CCF_AGENT_INSTANCE_RETENTION` | `720h` | Daemon (or unknown) instances not seen for this long are pruned. | +| `CCF_AGENT_INSTANCE_ONESHOT_RETENTION` | `24h` | One-shot (`daemon: false`) instances not seen for this long are pruned. | +| `CCF_AGENT_INSTANCE_PRUNE_ENABLED` | `true` | Schedules the prune job (needs the worker service). | +| `CCF_AGENT_INSTANCE_PRUNE_SCHEDULE` | `0 17 * * * *` | River cron (6 fields, seconds first) of the prune job. It runs at most hourly; a more frequent schedule is not honored. | +| `CCF_AGENT_MAX_INSTANCES` | `500` | Non-prunable instances per agent; see the cap above. | + +A value that is unset or not positive takes the default. From 6c120ea1a87b3d15482d964e393fec80d2f7553b Mon Sep 17 00:00:00 2001 From: "ccf-lisa[bot]" <286799724+ccf-lisa[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:07:21 -0300 Subject: [PATCH 4/5] fix(api): paginate the admin agent instance list GET /api/admin/agents/{id}/instances returned every instance's summary, about 3 MiB each in the worst case, so one agent credential could inflate it past 1 GiB. It now takes page (default 1) and limit (default 25, max 25; a larger limit is capped, as ParseParams does elsewhere) and returns one page. An invalid page or limit is a 400, as on the revision list. meta keeps desired-revision and counts, which cover all of the agent's instances (agentcfg.CountInstances, scalar columns only), and adds page, limit, total and total-pages. Swagger and the operator guide describe the paging and the per-page bound. Co-Authored-By: Claude Opus 5.5 --- docs/agent-remote-config.md | 9 +- docs/docs.go | 26 ++- docs/swagger.json | 26 ++- docs/swagger.yaml | 27 ++- internal/api/handler/agent_config.go | 69 +++--- .../agent_config_admin_integration_test.go | 205 +++++++++++++++++- ...ig_report_remote_config_regression_test.go | 2 +- 7 files changed, 319 insertions(+), 45 deletions(-) diff --git a/docs/agent-remote-config.md b/docs/agent-remote-config.md index d6c90f9c..14551942 100644 --- a/docs/agent-remote-config.md +++ b/docs/agent-remote-config.md @@ -43,7 +43,7 @@ Every admin route returns `400` for a malformed agent ID, `403` without the perm | `GET /api/admin/agents/{id}/config/revisions` | `200` revisions newest first, without overlays; `page` (default 1) and `limit` (default 50, max 100). | | `GET /api/admin/agents/{id}/config/revisions/{rev}` | `200` one revision; `404` unknown revision. | | `POST /api/admin/agents/{id}/config/revisions/{rev}/revert` | Saves revision `rev`'s overlay as the next revision (`revert-of` records it). Same `If-Match`, validation and status codes as `PUT`; the body (`{"comment": ...}`) is optional. | -| `GET /api/admin/agents/{id}/instances` | `200` every instance's summary with `meta.desired-revision` and `meta.counts`. Not paginated. | +| `GET /api/admin/agents/{id}/instances` | `200` one page of instance summaries, most recently seen first; `page` (default 1) and `limit` (default 25, max 25; a larger limit is capped). `meta.desired-revision` and `meta.counts` cover all of the agent's instances; `meta.page`, `meta.limit`, `meta.total` and `meta.total-pages` describe the page. `400` invalid `page` or `limit`. | | `GET /api/admin/agents/{id}/instances/{instanceId}` | `200` the summary plus the redacted `base` and `effective` configs; `404` unknown instance. | The CORS configuration allows the `If-Match` and `If-None-Match` request headers and exposes @@ -164,8 +164,11 @@ Deleting an agent deletes its instances and revisions. plugins, warnings and unsafe changes by count, their strings, the error text, hostname and version by length, and the reported `remote_config` (at most 100 `trusted_sources` and 100 `overridable_config_flags`, each at most 256 bytes encoded, dropped rather than cut; about -64 KiB in all). The instance list returns these summary columns for every instance, without -the base and effective configs; in the worst case that is about 3 MiB per instance. +64 KiB in all). The instance list returns these summary columns, without the base and +effective configs; in the worst case that is about 3 MiB of text per instance. So the list +is paginated at 25 instances, about 75 MiB at most (text that JSON escapes, such as `<` or +control characters, encodes up to six times larger), and its counts are computed from the +instances' status columns alone. ## Who sees secrets diff --git a/docs/docs.go b/docs/docs.go index 21a67517..99f210b0 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -528,7 +528,7 @@ const docTemplate = `{ }, "/admin/agents/{id}/instances": { "get": { - "description": "Summaries of the instances that reported or heartbeated with a config digest, with the derived status (pending and unknown are server-derived), sync status, staleness and counts. Base/effective configs are on the instance detail route.", + "description": "One page of summaries of the instances that reported or heartbeated with a config digest, most recently seen first, with the derived status (pending and unknown are server-derived), sync status and staleness. meta.counts and meta.desired-revision cover all of the agent's instances, not just the page; meta.page, meta.limit, meta.total and meta.total-pages describe the page. A limit above 25 is capped at 25, since one instance's summary can reach about 3 MiB. Base/effective configs are on the instance detail route.", "produces": [ "application/json" ], @@ -543,6 +543,18 @@ const docTemplate = `{ "name": "id", "in": "path", "required": true + }, + { + "type": "integer", + "description": "Page (default 1)", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "Page size (default 25, max 25)", + "name": "limit", + "in": "query" } ], "responses": { @@ -39812,6 +39824,18 @@ const docTemplate = `{ }, "desired-revision": { "type": "integer" + }, + "limit": { + "type": "integer" + }, + "page": { + "type": "integer" + }, + "total": { + "type": "integer" + }, + "total-pages": { + "type": "integer" } } }, diff --git a/docs/swagger.json b/docs/swagger.json index 2b5624e1..9d49722f 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -522,7 +522,7 @@ }, "/admin/agents/{id}/instances": { "get": { - "description": "Summaries of the instances that reported or heartbeated with a config digest, with the derived status (pending and unknown are server-derived), sync status, staleness and counts. Base/effective configs are on the instance detail route.", + "description": "One page of summaries of the instances that reported or heartbeated with a config digest, most recently seen first, with the derived status (pending and unknown are server-derived), sync status and staleness. meta.counts and meta.desired-revision cover all of the agent's instances, not just the page; meta.page, meta.limit, meta.total and meta.total-pages describe the page. A limit above 25 is capped at 25, since one instance's summary can reach about 3 MiB. Base/effective configs are on the instance detail route.", "produces": [ "application/json" ], @@ -537,6 +537,18 @@ "name": "id", "in": "path", "required": true + }, + { + "type": "integer", + "description": "Page (default 1)", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "Page size (default 25, max 25)", + "name": "limit", + "in": "query" } ], "responses": { @@ -39806,6 +39818,18 @@ }, "desired-revision": { "type": "integer" + }, + "limit": { + "type": "integer" + }, + "page": { + "type": "integer" + }, + "total": { + "type": "integer" + }, + "total-pages": { + "type": "integer" } } }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 431330ed..cd60b811 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -3060,6 +3060,14 @@ definitions: $ref: '#/definitions/handler.agentInstanceCounts' desired-revision: type: integer + limit: + type: integer + page: + type: integer + total: + type: integer + total-pages: + type: integer type: object handler.attachFilterResponsibilityRequest: properties: @@ -13253,16 +13261,27 @@ paths: - Agent Configuration /admin/agents/{id}/instances: get: - description: Summaries of the instances that reported or heartbeated with a - config digest, with the derived status (pending and unknown are server-derived), - sync status, staleness and counts. Base/effective configs are on the instance - detail route. + description: One page of summaries of the instances that reported or heartbeated + with a config digest, most recently seen first, with the derived status (pending + and unknown are server-derived), sync status and staleness. meta.counts and + meta.desired-revision cover all of the agent's instances, not just the page; + meta.page, meta.limit, meta.total and meta.total-pages describe the page. + A limit above 25 is capped at 25, since one instance's summary can reach about + 3 MiB. Base/effective configs are on the instance detail route. parameters: - description: Agent ID in: path name: id required: true type: string + - description: Page (default 1) + in: query + name: page + type: integer + - description: Page size (default 25, max 25) + in: query + name: limit + type: integer produces: - application/json responses: diff --git a/internal/api/handler/agent_config.go b/internal/api/handler/agent_config.go index aef8f951..8c0837e1 100644 --- a/internal/api/handler/agent_config.go +++ b/internal/api/handler/agent_config.go @@ -112,6 +112,7 @@ type agentInstanceDetail struct { Effective json.RawMessage `json:"effective" swaggertype:"object"` } +// agentInstanceCounts is agentcfg.InstanceCounts with JSON names (converted directly). type agentInstanceCounts struct { Total int `json:"total"` Fresh int `json:"fresh"` @@ -124,9 +125,15 @@ type agentInstanceCounts struct { Unknown int `json:"unknown"` } +// agentInstancesMeta is the meta of the instance list: the desired revision and the counts +// cover all of the agent's instances; page, limit, total and total-pages describe the page. type agentInstancesMeta struct { DesiredRevision int64 `json:"desired-revision"` Counts agentInstanceCounts `json:"counts"` + Page int `json:"page"` + Limit int `json:"limit"` + Total int64 `json:"total"` + TotalPages int `json:"total-pages"` } type configPreviewResponse struct { @@ -689,15 +696,17 @@ func (h *AgentConfigHandler) GetRevision(ctx echo.Context) error { // ListInstances godoc // // @Summary List an agent's instances -// @Description Summaries of the instances that reported or heartbeated with a config digest, with the derived status (pending and unknown are server-derived), sync status, staleness and counts. Base/effective configs are on the instance detail route. +// @Description One page of summaries of the instances that reported or heartbeated with a config digest, most recently seen first, with the derived status (pending and unknown are server-derived), sync status and staleness. meta.counts and meta.desired-revision cover all of the agent's instances, not just the page; meta.page, meta.limit, meta.total and meta.total-pages describe the page. A limit above 25 is capped at 25, since one instance's summary can reach about 3 MiB. Base/effective configs are on the instance detail route. // @Tags Agent Configuration // @Produce json -// @Param id path string true "Agent ID" -// @Success 200 {object} handler.GenericDataListResponse[handler.agentInstanceSummary]{meta=handler.agentInstancesMeta} -// @Failure 400 {object} api.Error -// @Failure 403 {object} api.Error -// @Failure 404 {object} api.Error -// @Failure 500 {object} api.Error +// @Param id path string true "Agent ID" +// @Param page query integer false "Page (default 1)" +// @Param limit query integer false "Page size (default 25, max 25)" +// @Success 200 {object} handler.GenericDataListResponse[handler.agentInstanceSummary]{meta=handler.agentInstancesMeta} +// @Failure 400 {object} api.Error +// @Failure 403 {object} api.Error +// @Failure 404 {object} api.Error +// @Failure 500 {object} api.Error // @Security OAuth2Password // @Router /admin/agents/{id}/instances [get] func (h *AgentConfigHandler) ListInstances(ctx echo.Context) error { @@ -705,44 +714,36 @@ func (h *AgentConfigHandler) ListInstances(ctx echo.Context) error { if agent == nil { return errResp } + pagination := service.PaginationConfig{DefaultLimit: agentcfg.InstancesPageLimit, MaxLimit: agentcfg.InstancesPageLimit} + params, err := pagination.ParseParams(ctx) + if err != nil { + return ctx.JSON(http.StatusBadRequest, api.NewError(err)) + } reqCtx := ctx.Request().Context() desired, err := h.svc.CurrentRevisionNumber(reqCtx, *agent.ID) if err != nil { return h.internalError(ctx, "load agent configuration", err) } - instances, err := h.svc.ListInstances(reqCtx, *agent.ID) + instances, total, err := h.svc.ListInstances(reqCtx, *agent.ID, *params) if err != nil { return h.internalError(ctx, "list instances", err) } now := h.svc.Now() + counts, err := h.svc.CountInstances(reqCtx, *agent.ID, desired, now) + if err != nil { + return h.internalError(ctx, "count instances", err) + } data := make([]agentInstanceSummary, 0, len(instances)) - meta := agentInstancesMeta{DesiredRevision: desired} - counts := &meta.Counts for _, inst := range instances { - s := h.instanceSummary(inst, desired, now) - data = append(data, s) - counts.Total++ - if s.Stale { - counts.Stale++ - } else { - counts.Fresh++ - } - switch s.SyncStatus { - case agentcfg.SyncInSync: - counts.InSync++ - case agentcfg.SyncOutOfSync: - counts.OutOfSync++ - } - switch s.Status { - case agentconfig.StatusPending: - counts.Pending++ - case agentconfig.StatusRejected: - counts.Rejected++ - case agentconfig.StatusFailed: - counts.Failed++ - case agentconfig.StatusUnknown: - counts.Unknown++ - } + data = append(data, h.instanceSummary(inst, desired, now)) + } + meta := agentInstancesMeta{ + DesiredRevision: desired, + Counts: agentInstanceCounts(counts), + Page: params.Page, + Limit: params.Limit, + Total: total, + TotalPages: max(1, int((total+int64(params.Limit)-1)/int64(params.Limit))), } return ctx.JSON(http.StatusOK, GenericDataListResponse[agentInstanceSummary]{Data: data, Meta: meta}) } diff --git a/internal/api/handler/agent_config_admin_integration_test.go b/internal/api/handler/agent_config_admin_integration_test.go index b789cac9..7d95403a 100644 --- a/internal/api/handler/agent_config_admin_integration_test.go +++ b/internal/api/handler/agent_config_admin_integration_test.go @@ -849,6 +849,11 @@ func (s *AgentConfigAdminIntegrationSuite) TestInstances() { InSync: 1, OutOfSync: 2, Pending: 1, Rejected: 1, Failed: 0, Unknown: 1, }, list.Meta.Counts) + s.Equal(1, list.Meta.Page) + s.Equal(agentcfg.InstancesPageLimit, list.Meta.Limit) + s.Equal(int64(5), list.Meta.Total) + s.Equal(1, list.Meta.TotalPages) + s.Len(list.Data, 5) byID := map[string]agentInstanceSummary{} for _, inst := range list.Data { @@ -931,7 +936,205 @@ func (s *AgentConfigAdminIntegrationSuite) TestInstances() { func (s *AgentConfigAdminIntegrationSuite) TestInstancesEmpty() { rec := s.call(http.MethodGet, s.path("/instances"), nil) s.Require().Equal(http.StatusOK, rec.Code, rec.Body.String()) - s.JSONEq(`{"data":[],"meta":{"desired-revision":0,"counts":{"total":0,"fresh":0,"stale":0,"in-sync":0,"out-of-sync":0,"pending":0,"rejected":0,"failed":0,"unknown":0}}}`, rec.Body.String()) + s.JSONEq(`{"data":[],"meta":{"desired-revision":0,"counts":{"total":0,"fresh":0,"stale":0,"in-sync":0,"out-of-sync":0,"pending":0,"rejected":0,"failed":0,"unknown":0},"page":1,"limit":25,"total":0,"total-pages":1}}`, rec.Body.String()) +} + +type acaInstanceList struct { + Data []agentInstanceSummary `json:"data"` + Meta agentInstancesMeta `json:"meta"` +} + +func (s *AgentConfigAdminIntegrationSuite) listInstances(query string) acaInstanceList { + rec := s.call(http.MethodGet, s.path("/instances"+query), nil) + s.Require().Equal(http.StatusOK, rec.Code, rec.Body.String()) + var list acaInstanceList + s.Require().NoError(json.Unmarshal(rec.Body.Bytes(), &list)) + return list +} + +// The instance list is paginated (review #476, fp 5d771b4b3d43): at most 25 instances per +// page, most recently seen first, while meta.counts cover every instance of the agent. +func (s *AgentConfigAdminIntegrationSuite) TestInstancesPagination() { + ctx := context.Background() + agentID := *s.agent.ID + s.save(`"0"`, `{"verbosity":1}`, 1) // desired revision 1 + + // Instance i was last seen i minutes ago, so instances 11 and up are stale. The first page + // (0-24) is in sync; the states that differ are all beyond it. + const n = 30 + want := make([]string, n) + for i := 0; i < n; i++ { + var id uuid.UUID + switch i { + case 25: + id = s.report(agentID, agentconfig.ModeApplySafe, func(r *agentconfig.Report) { + r.AttemptedRevision = acaI64(1) + r.Status = agentconfig.StatusRejected + r.Reason = agentconfig.ReasonUnsafeChanges + }) + case 26: + id = s.report(agentID, agentconfig.ModeApplyAll, func(r *agentconfig.Report) { + r.AttemptedRevision = acaI64(1) + r.Status = agentconfig.StatusFailed + }) + case 27: + id = s.report(agentID, agentconfig.ModeReport, nil) + case 28: + id = uuid.New() + digest := acaDigest + s.Require().NoError(s.svc.TouchFromHeartbeat(ctx, agentID, nil, id, acaI64(0), &digest)) + case 29: + id = s.report(agentID, agentconfig.ModeApplySafe, nil) // behind, not attempted: pending + default: + id = s.report(agentID, agentconfig.ModeApplySafe, func(r *agentconfig.Report) { r.AppliedRevision = acaI64(1) }) + } + s.makeStale(id, time.Duration(i)*time.Minute+time.Second) + want[i] = id.String() + } + ids := func(list acaInstanceList) []string { + out := make([]string, 0, len(list.Data)) + for _, inst := range list.Data { + out = append(out, inst.InstanceID) + } + return out + } + wantCounts := agentInstanceCounts{ + Total: n, Fresh: 10, Stale: n - 10, + InSync: 25, OutOfSync: 3, + Pending: 1, Rejected: 1, Failed: 1, Unknown: 1, + } + + first := s.listInstances("") + s.Equal(want[:25], ids(first), "default page: the 25 most recently seen") + s.Equal(agentInstancesMeta{DesiredRevision: 1, Counts: wantCounts, Page: 1, Limit: 25, Total: n, TotalPages: 2}, first.Meta, + "counts cover the instances beyond the page") + + second := s.listInstances("?page=2") + s.Equal(want[25:], ids(second)) + s.Equal(agentInstancesMeta{DesiredRevision: 1, Counts: wantCounts, Page: 2, Limit: 25, Total: n, TotalPages: 2}, second.Meta) + s.Equal(agentconfig.StatusRejected, second.Data[0].Status) + s.Equal(agentconfig.StatusPending, second.Data[4].Status) + + third := s.listInstances("?page=3&limit=10") + s.Equal(want[20:30], ids(third)) + s.Equal(3, third.Meta.TotalPages) + s.Equal(wantCounts, third.Meta.Counts) + + capped := s.listInstances("?limit=1000") + s.Equal(want[:25], ids(capped), "a limit over the maximum is capped, as ParseParams does") + s.Equal(25, capped.Meta.Limit) + + beyond := s.listInstances("?page=9") + s.Empty(beyond.Data) + s.Equal(9, beyond.Meta.Page) + s.Equal(int64(n), beyond.Meta.Total) + s.Equal(wantCounts, beyond.Meta.Counts) +} + +// acaInstanceSummaryBound is the documented bound on one listed instance's summary +// (agentcfg.InstancesPageLimit): about 3 MiB of text at the report caps of normalizeReport. +const acaInstanceSummaryBound = 3 << 20 + +// acaWorstCaseReport returns a report over every cap normalizeReport applies to the summary +// columns (warnings, unsafe changes, plugins, remote-config, error, hostname, version), +// normalized as the report route stores it. Its text is plain ASCII, which JSON encodes 1:1. +func (s *AgentConfigAdminIntegrationSuite) acaWorstCaseReport() agentconfig.Report { + long := func(c string, n int) string { return strings.Repeat(c, n+16) } // over the cap + errText := long("e", maxReportErrorBytes) + r := agentconfig.Report{ + Hostname: long("h", maxReportHostnameLen), + AgentVersion: long("v", maxReportAgentVersionLen), + Mode: agentconfig.ModeApplySafe, + Daemon: true, + AppliedRevision: acaI64(0), + Status: agentconfig.StatusApplied, + Error: &errText, + Base: json.RawMessage(`{}`), + Effective: json.RawMessage(`{}`), + EffectiveDigest: acaDigest, + RemoteConfig: &agentconfig.RemoteConfig{ + Mode: long("m", maxReportRemoteModeLen), + PollInterval: long("p", maxReportRemotePollIntervalLen), + }, + } + for i := 0; i < maxReportWarnings+1; i++ { + r.Warnings = append(r.Warnings, agentconfig.FieldError{ + Path: long("w", maxReportWarningPathBytes), Code: agentconfig.FieldCodeUnknownField, Message: long("m", maxReportWarningMessageBytes), + }) + } + for i := 0; i < maxReportUnsafe+1; i++ { + r.Unsafe = append(r.Unsafe, agentconfig.Change{ + Path: long("u", maxReportChangePathBytes), Safety: agentconfig.Unsafe, + Reason: agentconfig.ChangeReasonUntrustedSource, Value: long("c", maxReportChangeValueBytes), + }) + } + for i := 0; i < maxReportPlugins+1; i++ { + r.Plugins = append(r.Plugins, agentconfig.PluginReport{ + Name: long("n", maxReportPluginNameLen), Source: long("s", maxReportPluginSourceLen), LibVersion: long("l", maxReportPluginLibVersionLen), + }) + } + entry := strings.Repeat("t", maxReportRemoteEntryBytes-2) // at the cap once JSON-encoded + for i := 0; i < maxReportRemoteListEntries+1; i++ { + r.RemoteConfig.TrustedSources = append(r.RemoteConfig.TrustedSources, entry) + r.RemoteConfig.OverridableConfigFlags = append(r.RemoteConfig.OverridableConfigFlags, entry) + } + scrubbed, err := normalizeReport(&r) + s.Require().NoError(err) + s.Require().False(scrubbed, "nothing in the report looks like a secret") + s.Require().True(r.Truncated) + return r +} + +// A page of worst-case instances stays within InstancesPageLimit times the documented +// per-instance bound (review #476, fp 5d771b4b3d43): an agent credential can no longer +// inflate the list with its instance count. +func (s *AgentConfigAdminIntegrationSuite) TestInstancesWorstCasePageSize() { + agentID := *s.agent.ID + report := s.acaWorstCaseReport() + const n = agentcfg.InstancesPageLimit + 1 + for i := 0; i < n; i++ { + s.Require().NoError(s.svc.UpsertReport(context.Background(), agentID, nil, uuid.New(), report)) + } + + rec := s.call(http.MethodGet, s.path("/instances"), nil) + s.Require().Equal(http.StatusOK, rec.Code) + size := rec.Body.Len() + s.T().Logf("a page of %d worst-case instances: %d bytes", agentcfg.InstancesPageLimit, size) + s.LessOrEqual(size, agentcfg.InstancesPageLimit*acaInstanceSummaryBound+64<<10, + "a page of %d worst-case instances encodes to %d bytes", agentcfg.InstancesPageLimit, size) + s.Greater(size, agentcfg.InstancesPageLimit*(acaInstanceSummaryBound*9/10), + "the instances are near the bound, so the check above is meaningful") + + var list acaInstanceList + s.Require().NoError(json.Unmarshal(rec.Body.Bytes(), &list)) + s.Require().Len(list.Data, agentcfg.InstancesPageLimit) + s.Equal(int64(n), list.Meta.Total) + s.Equal(n, list.Meta.Counts.Total) + for _, inst := range list.Data { + s.Len(inst.Warnings, maxReportWarnings) + s.Len(inst.Unsafe, maxReportUnsafe) + s.Len(inst.Plugins, maxReportPlugins) + s.True(inst.Truncated) + } +} + +// An invalid page or limit is a 400, as on the revision list. +func (s *AgentConfigAdminIntegrationSuite) TestInstancesBadPagination() { + s.report(*s.agent.ID, agentconfig.ModeApplySafe, nil) + for q, msg := range map[string]string{ + "page=0": "page must be greater than 0", + "page=-1": "page must be greater than 0", + "page=x": "invalid page parameter: x", + "page=1.5": "invalid page parameter: 1.5", + "limit=0": "limit must be greater than 0", + "limit=-5": "limit must be greater than 0", + "limit=x": "invalid limit parameter: x", + "limit=1e3": "invalid limit parameter: 1e3", + } { + rec := s.call(http.MethodGet, s.path("/instances?"+q), nil) + s.Require().Equal(http.StatusBadRequest, rec.Code, "%s: %s", q, rec.Body.String()) + s.Equal(msg, s.errorBody(rec), q) + } } // ---- Agent deletion ---- diff --git a/internal/api/handler/agent_config_report_remote_config_regression_test.go b/internal/api/handler/agent_config_report_remote_config_regression_test.go index 3b571018..645a3e07 100644 --- a/internal/api/handler/agent_config_report_remote_config_regression_test.go +++ b/internal/api/handler/agent_config_report_remote_config_regression_test.go @@ -12,7 +12,7 @@ import ( ) // maxStoredRemoteConfigBytes is the most remote-config JSON a stored report may keep. The -// block is a summary column returned for every instance by the unpaginated instance list. +// block is a summary column returned for every instance on a page of the instance list. const maxStoredRemoteConfigBytes = 64 << 10 func regressionReport(rc *agentconfig.RemoteConfig) agentconfig.Report { From f8cb15aa0b8cae5426b97b022a62e47fd8e58ea9 Mon Sep 17 00:00:00 2001 From: "ccf-lisa[bot]" <286799724+ccf-lisa[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:46:25 -0300 Subject: [PATCH 5/5] fix(api): test the instance page bound with escaped report text TestInstancesWorstCasePageSize now builds its reports with capsReport and normalizeReport, over every cap, once with plain text and once with text that JSON escapes ('<', '&', control characters), and checks that a page of 25 stays within 25 times the per-report summary budget plus the fixed fields. The swagger description and the operator guide describe the bound after escaping. Co-Authored-By: Claude Opus 5.5 --- docs/agent-remote-config.md | 19 ++- docs/docs.go | 2 +- docs/swagger.json | 2 +- docs/swagger.yaml | 2 +- internal/api/handler/agent_config.go | 2 +- .../agent_config_admin_integration_test.go | 135 +++++++----------- 6 files changed, 69 insertions(+), 93 deletions(-) diff --git a/docs/agent-remote-config.md b/docs/agent-remote-config.md index 14551942..603bf9ee 100644 --- a/docs/agent-remote-config.md +++ b/docs/agent-remote-config.md @@ -161,13 +161,18 @@ agent). Existing instances keep reporting. Deleting an agent deletes its instances and revisions. **Report bounds.** The API caps what a report stores and marks the instance `truncated`: -plugins, warnings and unsafe changes by count, their strings, the error text, hostname and -version by length, and the reported `remote_config` (at most 100 `trusted_sources` and 100 -`overridable_config_flags`, each at most 256 bytes encoded, dropped rather than cut; about -64 KiB in all). The instance list returns these summary columns, without the base and -effective configs; in the worst case that is about 3 MiB of text per instance. So the list -is paginated at 25 instances, about 75 MiB at most (text that JSON escapes, such as `<` or -control characters, encodes up to six times larger), and its counts are computed from the +plugins, warnings and unsafe changes by count, every one of their strings (warning codes and +change safety and reason included, cut to 64 bytes rather than rejected, so a newer agent's +values still fit), the error text, hostname and version by length, and the reported +`remote_config` (at most 100 `trusted_sources` and 100 `overridable_config_flags`, each at +most 256 bytes encoded, dropped rather than cut; about 64 KiB in all). On top of these caps, +the summary fields as a whole (hostname, version, error, warnings, unsafe changes, plugins and +`remote_config`) are kept within 3 MiB as the instance list encodes them, JSON escaping +included (`<`, `&` and control characters take six bytes each): over that, the API drops the +last entries of the largest list and cuts the error text until the report fits. A plain-text +report at every cap is about 2.9 MB and is stored unchanged. The instance list returns these +summary columns, without the base and effective configs, so with pages of 25 instances a page +is about 75 MiB at most, whatever the reports contain. Its counts are computed from the instances' status columns alone. ## Who sees secrets diff --git a/docs/docs.go b/docs/docs.go index 99f210b0..e38ab28a 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -528,7 +528,7 @@ const docTemplate = `{ }, "/admin/agents/{id}/instances": { "get": { - "description": "One page of summaries of the instances that reported or heartbeated with a config digest, most recently seen first, with the derived status (pending and unknown are server-derived), sync status and staleness. meta.counts and meta.desired-revision cover all of the agent's instances, not just the page; meta.page, meta.limit, meta.total and meta.total-pages describe the page. A limit above 25 is capped at 25, since one instance's summary can reach about 3 MiB. Base/effective configs are on the instance detail route.", + "description": "One page of summaries of the instances that reported or heartbeated with a config digest, most recently seen first, with the derived status (pending and unknown are server-derived), sync status and staleness. meta.counts and meta.desired-revision cover all of the agent's instances, not just the page; meta.page, meta.limit, meta.total and meta.total-pages describe the page. A limit above 25 is capped at 25, since one instance's summary can reach about 3 MiB once encoded. Base/effective configs are on the instance detail route.", "produces": [ "application/json" ], diff --git a/docs/swagger.json b/docs/swagger.json index 9d49722f..e66a80ea 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -522,7 +522,7 @@ }, "/admin/agents/{id}/instances": { "get": { - "description": "One page of summaries of the instances that reported or heartbeated with a config digest, most recently seen first, with the derived status (pending and unknown are server-derived), sync status and staleness. meta.counts and meta.desired-revision cover all of the agent's instances, not just the page; meta.page, meta.limit, meta.total and meta.total-pages describe the page. A limit above 25 is capped at 25, since one instance's summary can reach about 3 MiB. Base/effective configs are on the instance detail route.", + "description": "One page of summaries of the instances that reported or heartbeated with a config digest, most recently seen first, with the derived status (pending and unknown are server-derived), sync status and staleness. meta.counts and meta.desired-revision cover all of the agent's instances, not just the page; meta.page, meta.limit, meta.total and meta.total-pages describe the page. A limit above 25 is capped at 25, since one instance's summary can reach about 3 MiB once encoded. Base/effective configs are on the instance detail route.", "produces": [ "application/json" ], diff --git a/docs/swagger.yaml b/docs/swagger.yaml index cd60b811..f67629e2 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -13267,7 +13267,7 @@ paths: meta.desired-revision cover all of the agent's instances, not just the page; meta.page, meta.limit, meta.total and meta.total-pages describe the page. A limit above 25 is capped at 25, since one instance's summary can reach about - 3 MiB. Base/effective configs are on the instance detail route. + 3 MiB once encoded. Base/effective configs are on the instance detail route. parameters: - description: Agent ID in: path diff --git a/internal/api/handler/agent_config.go b/internal/api/handler/agent_config.go index 8c0837e1..c7d92bf7 100644 --- a/internal/api/handler/agent_config.go +++ b/internal/api/handler/agent_config.go @@ -696,7 +696,7 @@ func (h *AgentConfigHandler) GetRevision(ctx echo.Context) error { // ListInstances godoc // // @Summary List an agent's instances -// @Description One page of summaries of the instances that reported or heartbeated with a config digest, most recently seen first, with the derived status (pending and unknown are server-derived), sync status and staleness. meta.counts and meta.desired-revision cover all of the agent's instances, not just the page; meta.page, meta.limit, meta.total and meta.total-pages describe the page. A limit above 25 is capped at 25, since one instance's summary can reach about 3 MiB. Base/effective configs are on the instance detail route. +// @Description One page of summaries of the instances that reported or heartbeated with a config digest, most recently seen first, with the derived status (pending and unknown are server-derived), sync status and staleness. meta.counts and meta.desired-revision cover all of the agent's instances, not just the page; meta.page, meta.limit, meta.total and meta.total-pages describe the page. A limit above 25 is capped at 25, since one instance's summary can reach about 3 MiB once encoded. Base/effective configs are on the instance detail route. // @Tags Agent Configuration // @Produce json // @Param id path string true "Agent ID" diff --git a/internal/api/handler/agent_config_admin_integration_test.go b/internal/api/handler/agent_config_admin_integration_test.go index 7d95403a..b965f2bb 100644 --- a/internal/api/handler/agent_config_admin_integration_test.go +++ b/internal/api/handler/agent_config_admin_integration_test.go @@ -1031,90 +1031,61 @@ func (s *AgentConfigAdminIntegrationSuite) TestInstancesPagination() { s.Equal(wantCounts, beyond.Meta.Counts) } -// acaInstanceSummaryBound is the documented bound on one listed instance's summary -// (agentcfg.InstancesPageLimit): about 3 MiB of text at the report caps of normalizeReport. -const acaInstanceSummaryBound = 3 << 20 - -// acaWorstCaseReport returns a report over every cap normalizeReport applies to the summary -// columns (warnings, unsafe changes, plugins, remote-config, error, hostname, version), -// normalized as the report route stores it. Its text is plain ASCII, which JSON encodes 1:1. -func (s *AgentConfigAdminIntegrationSuite) acaWorstCaseReport() agentconfig.Report { - long := func(c string, n int) string { return strings.Repeat(c, n+16) } // over the cap - errText := long("e", maxReportErrorBytes) - r := agentconfig.Report{ - Hostname: long("h", maxReportHostnameLen), - AgentVersion: long("v", maxReportAgentVersionLen), - Mode: agentconfig.ModeApplySafe, - Daemon: true, - AppliedRevision: acaI64(0), - Status: agentconfig.StatusApplied, - Error: &errText, - Base: json.RawMessage(`{}`), - Effective: json.RawMessage(`{}`), - EffectiveDigest: acaDigest, - RemoteConfig: &agentconfig.RemoteConfig{ - Mode: long("m", maxReportRemoteModeLen), - PollInterval: long("p", maxReportRemotePollIntervalLen), - }, - } - for i := 0; i < maxReportWarnings+1; i++ { - r.Warnings = append(r.Warnings, agentconfig.FieldError{ - Path: long("w", maxReportWarningPathBytes), Code: agentconfig.FieldCodeUnknownField, Message: long("m", maxReportWarningMessageBytes), - }) - } - for i := 0; i < maxReportUnsafe+1; i++ { - r.Unsafe = append(r.Unsafe, agentconfig.Change{ - Path: long("u", maxReportChangePathBytes), Safety: agentconfig.Unsafe, - Reason: agentconfig.ChangeReasonUntrustedSource, Value: long("c", maxReportChangeValueBytes), - }) - } - for i := 0; i < maxReportPlugins+1; i++ { - r.Plugins = append(r.Plugins, agentconfig.PluginReport{ - Name: long("n", maxReportPluginNameLen), Source: long("s", maxReportPluginSourceLen), LibVersion: long("l", maxReportPluginLibVersionLen), - }) - } - entry := strings.Repeat("t", maxReportRemoteEntryBytes-2) // at the cap once JSON-encoded - for i := 0; i < maxReportRemoteListEntries+1; i++ { - r.RemoteConfig.TrustedSources = append(r.RemoteConfig.TrustedSources, entry) - r.RemoteConfig.OverridableConfigFlags = append(r.RemoteConfig.OverridableConfigFlags, entry) - } - scrubbed, err := normalizeReport(&r) - s.Require().NoError(err) - s.Require().False(scrubbed, "nothing in the report looks like a secret") - s.Require().True(r.Truncated) - return r -} - -// A page of worst-case instances stays within InstancesPageLimit times the documented -// per-instance bound (review #476, fp 5d771b4b3d43): an agent credential can no longer -// inflate the list with its instance count. +// acaInstanceFixedBytes bounds what a listed instance's summary holds besides the budgeted +// report fields (maxReportSummaryEncodedBytes): ids, times, statuses, digests and keys. +const acaInstanceFixedBytes = 2 << 10 + +// A page of worst-case instances stays within InstancesPageLimit times the per-instance +// budget (review #476/#480, fp 5d771b4b3d43), however many instances the agent has and +// whatever the reports contain: plain text at every cap, or text that JSON escapes ('<', '&' +// and control characters encode to six bytes each). The reports go through normalizeReport, +// as the report route stores them. func (s *AgentConfigAdminIntegrationSuite) TestInstancesWorstCasePageSize() { - agentID := *s.agent.ID - report := s.acaWorstCaseReport() - const n = agentcfg.InstancesPageLimit + 1 - for i := 0; i < n; i++ { - s.Require().NoError(s.svc.UpsertReport(context.Background(), agentID, nil, uuid.New(), report)) - } - - rec := s.call(http.MethodGet, s.path("/instances"), nil) - s.Require().Equal(http.StatusOK, rec.Code) - size := rec.Body.Len() - s.T().Logf("a page of %d worst-case instances: %d bytes", agentcfg.InstancesPageLimit, size) - s.LessOrEqual(size, agentcfg.InstancesPageLimit*acaInstanceSummaryBound+64<<10, - "a page of %d worst-case instances encodes to %d bytes", agentcfg.InstancesPageLimit, size) - s.Greater(size, agentcfg.InstancesPageLimit*(acaInstanceSummaryBound*9/10), - "the instances are near the bound, so the check above is meaningful") + for _, c := range []struct { + name, fill string + escaped bool + }{ + {"plain", "x", false}, + {"escaped", "<&\x01", true}, + } { + agent, err := s.CreateAgent("worst-case-" + c.name) + s.Require().NoError(err) + report := capsReport(c.fill, 1) // over every count and byte cap + _, err = normalizeReport(&report) + s.Require().NoError(err) + s.Require().True(report.Truncated) + const n = agentcfg.InstancesPageLimit + 1 + for i := 0; i < n; i++ { + s.Require().NoError(s.svc.UpsertReport(context.Background(), *agent.ID, nil, uuid.New(), report)) + } - var list acaInstanceList - s.Require().NoError(json.Unmarshal(rec.Body.Bytes(), &list)) - s.Require().Len(list.Data, agentcfg.InstancesPageLimit) - s.Equal(int64(n), list.Meta.Total) - s.Equal(n, list.Meta.Counts.Total) - for _, inst := range list.Data { - s.Len(inst.Warnings, maxReportWarnings) - s.Len(inst.Unsafe, maxReportUnsafe) - s.Len(inst.Plugins, maxReportPlugins) - s.True(inst.Truncated) + rec := s.call(http.MethodGet, s.agentPath(*agent.ID, "/instances"), nil) + s.Require().Equal(http.StatusOK, rec.Code, c.name) + size := rec.Body.Len() + s.T().Logf("%s: a page of %d worst-case instances encodes to %d bytes", c.name, agentcfg.InstancesPageLimit, size) + s.LessOrEqual(size, agentcfg.InstancesPageLimit*(maxReportSummaryEncodedBytes+acaInstanceFixedBytes)+64<<10, c.name) + s.Greater(size, agentcfg.InstancesPageLimit*(maxReportSummaryEncodedBytes*9/10), + "%s: the instances are near the budget, so the check above is meaningful", c.name) + + var list acaInstanceList + s.Require().NoError(json.Unmarshal(rec.Body.Bytes(), &list)) + s.Require().Len(list.Data, agentcfg.InstancesPageLimit, c.name) + s.Equal(int64(n), list.Meta.Total) + s.Equal(n, list.Meta.Counts.Total) + for _, inst := range list.Data { + s.True(inst.Truncated) + if c.escaped { + s.NotEmpty(inst.Warnings) + s.NotEmpty(inst.Unsafe) + s.NotEmpty(inst.Plugins) + s.Less(len(inst.Warnings)+len(inst.Unsafe)+len(inst.Plugins), maxReportWarnings+maxReportUnsafe+maxReportPlugins, + "escaped text is over the budget, so entries are dropped") + } else { + s.Len(inst.Warnings, maxReportWarnings, "plain text at the caps fits the budget") + s.Len(inst.Unsafe, maxReportUnsafe) + s.Len(inst.Plugins, maxReportPlugins) + } + } } }