From d98c799f1dd8511de0c1a645f11278d42839b20e Mon Sep 17 00:00:00 2001 From: "ccf-lisa[bot]" <286799724+ccf-lisa[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:15:13 -0300 Subject: [PATCH 1/4] feat(api): agent instance config reports Twelfth layer of the agent remote-configuration stack (split from #465): PUT /api/agent/instances/{instanceId}/config-report stores an instance's mode, applied/attempted revision, status, redacted base/effective configs, digest, plugins, unsafe changes and warnings. The server validates and bounds the report, rejects NUL characters, re-redacts base/effective, masks secrets in free-text fields (error, warnings, plugin sources, unsafe values, remote-config) and returns 409 at the instance cap. The JSON body reader accepts application/json with parameters (415/413 otherwise). Co-Authored-By: Claude Opus 5.5 --- docs/docs.go | 255 ++++++++++++ docs/swagger.json | 255 ++++++++++++ docs/swagger.yaml | 190 +++++++++ internal/api/handler/agent_config_body.go | 51 +++ internal/api/handler/agent_config_sync.go | 295 +++++++++++++- .../agent_config_sync_integration_test.go | 385 ++++++++++++++++++ .../api/handler/agent_config_sync_test.go | 358 ++++++++++++++++ 7 files changed, 1787 insertions(+), 2 deletions(-) create mode 100644 internal/api/handler/agent_config_sync_test.go diff --git a/docs/docs.go b/docs/docs.go index 9aafb6bd..1b96cf28 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2701,6 +2701,88 @@ const docTemplate = `{ } } }, + "/agent/instances/{instanceId}/config-report": { + "put": { + "description": "Stores the authenticated agent instance's config report: mode, applied/attempted revision, status (applied, rejected, failed or not-applicable; the server derives pending and unknown), the redacted base and effective configs (snake_case), the effective digest, plugins (with their agent-library version), unsafe changes, warnings and the normalized local remote_config block. The server re-redacts base and effective as a best effort, replaces error, warning messages, plugin sources, unsafe change values and remote-config strings that contain a secret with ••••, and stores effective-digest as sent. Long warning messages and unsafe lists are truncated (truncated=true). A NUL character anywhere is a 400. Body limit 4 MiB. A 409 means the per-agent instance cap is reached; back off.", + "consumes": [ + "application/json" + ], + "tags": [ + "Agents" + ], + "summary": "Report this instance's effective configuration", + "parameters": [ + { + "type": "string", + "description": "Agent instance ID (UUID)", + "name": "instanceId", + "in": "path", + "required": true + }, + { + "description": "Config report", + "name": "report", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/agentconfig.Report" + } + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "415": { + "description": "Unsupported Media Type", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.Error" + } + } + }, + "security": [ + { + "OAuth2Password": [] + } + ] + } + }, "/agent/risk-templates/batch": { "post": { "description": "Reconcile the full set of risk templates for a (plugin-id, policy-package) scope.\nCreates, updates, and deletes templates atomically. Templates not present in the payload are always deleted.", @@ -34384,6 +34466,42 @@ const docTemplate = `{ } }, "definitions": { + "agentconfig.Change": { + "type": "object", + "properties": { + "path": { + "description": "RFC 6901 pointer", + "type": "string" + }, + "reason": { + "type": "string" + }, + "safety": { + "$ref": "#/definitions/agentconfig.Safety" + }, + "value": { + "description": "the source / env name that triggered the class", + "type": "string" + } + } + }, + "agentconfig.FieldError": { + "type": "object", + "properties": { + "code": { + "description": "stable machine code (R43), one of FieldCode*", + "type": "string" + }, + "message": { + "description": "human text", + "type": "string" + }, + "path": { + "description": "RFC 6901 pointer into the snake_case config (\"\" = root)", + "type": "string" + } + } + }, "agentconfig.OverlayDocument": { "type": "object", "properties": { @@ -34399,6 +34517,143 @@ const docTemplate = `{ } } }, + "agentconfig.PluginReport": { + "type": "object", + "properties": { + "lib-version": { + "description": "LibVersion is the version of github.com/compliance-framework/agent the plugin binary\nwas built with, from its Go build info. Empty when unknown: no build info, or a\nreplace or devel build.", + "type": "string" + }, + "name": { + "description": "the plugin's key under plugins in the config", + "type": "string" + }, + "source": { + "description": "the configured source", + "type": "string" + } + } + }, + "agentconfig.RemoteConfig": { + "type": "object", + "properties": { + "allow_local_sources": { + "description": "default false", + "type": "boolean" + }, + "mode": { + "description": "Mode is off, report, apply_safe or apply_all. Unset means report for an agent with\ncredentials (it reports but never applies), off without (see Normalize).", + "type": "string" + }, + "overridable_config_flags": { + "description": "default []", + "type": "array", + "items": { + "type": "string" + } + }, + "poll_interval": { + "type": "string" + }, + "trusted_sources": { + "description": "default []", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "agentconfig.Report": { + "type": "object", + "properties": { + "agent-version": { + "description": "\u003c= 64", + "type": "string" + }, + "applied-revision": { + "type": "integer" + }, + "attempted-revision": { + "type": "integer" + }, + "base": { + "type": "object" + }, + "daemon": { + "description": "false = one-shot run; pruned after 24h (R10, R37)", + "type": "boolean" + }, + "effective": { + "type": "object" + }, + "effective-digest": { + "type": "string" + }, + "error": { + "description": "\u003c= 8 KiB, truncated server-side", + "type": "string" + }, + "hostname": { + "description": "\u003c= 255", + "type": "string" + }, + "mode": { + "type": "string" + }, + "plugins": { + "description": "Plugins are the instance's plugins and the agent library each was built with (R76),\nso the UI can show policy compatibility before a save. Older agents omit it.", + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.PluginReport" + } + }, + "reason": { + "type": "string" + }, + "remote-config": { + "description": "normalized; snake_case inside", + "allOf": [ + { + "$ref": "#/definitions/agentconfig.RemoteConfig" + } + ] + }, + "status": { + "type": "string" + }, + "truncated": { + "description": "agent dropped/trimmed parts to fit MaxReportBytes (R10)", + "type": "boolean" + }, + "unsafe": { + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.Change" + } + }, + "warnings": { + "description": "R41: tolerated file-origin problems", + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.FieldError" + } + } + } + }, + "agentconfig.Safety": { + "type": "string", + "enum": [ + "safe", + "unsafe", + "forbidden" + ], + "x-enum-varnames": [ + "Safe", + "Unsafe", + "Forbidden" + ] + }, "api.Error": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 112a3afe..d3fcbf07 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2695,6 +2695,88 @@ } } }, + "/agent/instances/{instanceId}/config-report": { + "put": { + "description": "Stores the authenticated agent instance's config report: mode, applied/attempted revision, status (applied, rejected, failed or not-applicable; the server derives pending and unknown), the redacted base and effective configs (snake_case), the effective digest, plugins (with their agent-library version), unsafe changes, warnings and the normalized local remote_config block. The server re-redacts base and effective as a best effort, replaces error, warning messages, plugin sources, unsafe change values and remote-config strings that contain a secret with ••••, and stores effective-digest as sent. Long warning messages and unsafe lists are truncated (truncated=true). A NUL character anywhere is a 400. Body limit 4 MiB. A 409 means the per-agent instance cap is reached; back off.", + "consumes": [ + "application/json" + ], + "tags": [ + "Agents" + ], + "summary": "Report this instance's effective configuration", + "parameters": [ + { + "type": "string", + "description": "Agent instance ID (UUID)", + "name": "instanceId", + "in": "path", + "required": true + }, + { + "description": "Config report", + "name": "report", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/agentconfig.Report" + } + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "415": { + "description": "Unsupported Media Type", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/api.Error" + } + } + }, + "security": [ + { + "OAuth2Password": [] + } + ] + } + }, "/agent/risk-templates/batch": { "post": { "description": "Reconcile the full set of risk templates for a (plugin-id, policy-package) scope.\nCreates, updates, and deletes templates atomically. Templates not present in the payload are always deleted.", @@ -34378,6 +34460,42 @@ } }, "definitions": { + "agentconfig.Change": { + "type": "object", + "properties": { + "path": { + "description": "RFC 6901 pointer", + "type": "string" + }, + "reason": { + "type": "string" + }, + "safety": { + "$ref": "#/definitions/agentconfig.Safety" + }, + "value": { + "description": "the source / env name that triggered the class", + "type": "string" + } + } + }, + "agentconfig.FieldError": { + "type": "object", + "properties": { + "code": { + "description": "stable machine code (R43), one of FieldCode*", + "type": "string" + }, + "message": { + "description": "human text", + "type": "string" + }, + "path": { + "description": "RFC 6901 pointer into the snake_case config (\"\" = root)", + "type": "string" + } + } + }, "agentconfig.OverlayDocument": { "type": "object", "properties": { @@ -34393,6 +34511,143 @@ } } }, + "agentconfig.PluginReport": { + "type": "object", + "properties": { + "lib-version": { + "description": "LibVersion is the version of github.com/compliance-framework/agent the plugin binary\nwas built with, from its Go build info. Empty when unknown: no build info, or a\nreplace or devel build.", + "type": "string" + }, + "name": { + "description": "the plugin's key under plugins in the config", + "type": "string" + }, + "source": { + "description": "the configured source", + "type": "string" + } + } + }, + "agentconfig.RemoteConfig": { + "type": "object", + "properties": { + "allow_local_sources": { + "description": "default false", + "type": "boolean" + }, + "mode": { + "description": "Mode is off, report, apply_safe or apply_all. Unset means report for an agent with\ncredentials (it reports but never applies), off without (see Normalize).", + "type": "string" + }, + "overridable_config_flags": { + "description": "default []", + "type": "array", + "items": { + "type": "string" + } + }, + "poll_interval": { + "type": "string" + }, + "trusted_sources": { + "description": "default []", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "agentconfig.Report": { + "type": "object", + "properties": { + "agent-version": { + "description": "\u003c= 64", + "type": "string" + }, + "applied-revision": { + "type": "integer" + }, + "attempted-revision": { + "type": "integer" + }, + "base": { + "type": "object" + }, + "daemon": { + "description": "false = one-shot run; pruned after 24h (R10, R37)", + "type": "boolean" + }, + "effective": { + "type": "object" + }, + "effective-digest": { + "type": "string" + }, + "error": { + "description": "\u003c= 8 KiB, truncated server-side", + "type": "string" + }, + "hostname": { + "description": "\u003c= 255", + "type": "string" + }, + "mode": { + "type": "string" + }, + "plugins": { + "description": "Plugins are the instance's plugins and the agent library each was built with (R76),\nso the UI can show policy compatibility before a save. Older agents omit it.", + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.PluginReport" + } + }, + "reason": { + "type": "string" + }, + "remote-config": { + "description": "normalized; snake_case inside", + "allOf": [ + { + "$ref": "#/definitions/agentconfig.RemoteConfig" + } + ] + }, + "status": { + "type": "string" + }, + "truncated": { + "description": "agent dropped/trimmed parts to fit MaxReportBytes (R10)", + "type": "boolean" + }, + "unsafe": { + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.Change" + } + }, + "warnings": { + "description": "R41: tolerated file-origin problems", + "type": "array", + "items": { + "$ref": "#/definitions/agentconfig.FieldError" + } + } + } + }, + "agentconfig.Safety": { + "type": "string", + "enum": [ + "safe", + "unsafe", + "forbidden" + ], + "x-enum-varnames": [ + "Safe", + "Unsafe", + "Forbidden" + ] + }, "api.Error": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index fb3c3c6c..a1671e9e 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2,6 +2,31 @@ basePath: /api consumes: - application/json definitions: + agentconfig.Change: + properties: + path: + description: RFC 6901 pointer + type: string + reason: + type: string + safety: + $ref: '#/definitions/agentconfig.Safety' + value: + description: the source / env name that triggered the class + type: string + type: object + agentconfig.FieldError: + properties: + code: + description: stable machine code (R43), one of FieldCode* + type: string + message: + description: human text + type: string + path: + description: RFC 6901 pointer into the snake_case config ("" = root) + type: string + type: object agentconfig.OverlayDocument: properties: created-at: @@ -12,6 +37,108 @@ definitions: revision: type: integer type: object + agentconfig.PluginReport: + properties: + lib-version: + description: |- + LibVersion is the version of github.com/compliance-framework/agent the plugin binary + was built with, from its Go build info. Empty when unknown: no build info, or a + replace or devel build. + type: string + name: + description: the plugin's key under plugins in the config + type: string + source: + description: the configured source + type: string + type: object + agentconfig.RemoteConfig: + properties: + allow_local_sources: + description: default false + type: boolean + mode: + description: |- + Mode is off, report, apply_safe or apply_all. Unset means report for an agent with + credentials (it reports but never applies), off without (see Normalize). + type: string + overridable_config_flags: + description: default [] + items: + type: string + type: array + poll_interval: + type: string + trusted_sources: + description: default [] + items: + type: string + type: array + type: object + agentconfig.Report: + properties: + agent-version: + description: <= 64 + type: string + applied-revision: + type: integer + attempted-revision: + type: integer + base: + type: object + daemon: + description: false = one-shot run; pruned after 24h (R10, R37) + type: boolean + effective: + type: object + effective-digest: + type: string + error: + description: <= 8 KiB, truncated server-side + type: string + hostname: + description: <= 255 + type: string + mode: + type: string + plugins: + description: |- + Plugins are the instance's plugins and the agent library each was built with (R76), + so the UI can show policy compatibility before a save. Older agents omit it. + items: + $ref: '#/definitions/agentconfig.PluginReport' + type: array + reason: + type: string + remote-config: + allOf: + - $ref: '#/definitions/agentconfig.RemoteConfig' + description: normalized; snake_case inside + status: + type: string + truncated: + description: agent dropped/trimmed parts to fit MaxReportBytes (R10) + type: boolean + unsafe: + items: + $ref: '#/definitions/agentconfig.Change' + type: array + warnings: + description: 'R41: tolerated file-origin problems' + items: + $ref: '#/definitions/agentconfig.FieldError' + type: array + type: object + agentconfig.Safety: + enum: + - safe + - unsafe + - forbidden + type: string + x-enum-varnames: + - Safe + - Unsafe + - Forbidden api.Error: properties: errors: @@ -14212,6 +14339,69 @@ paths: summary: Get Heartbeat Metrics Over Time tags: - Heartbeat + /agent/instances/{instanceId}/config-report: + put: + consumes: + - application/json + description: 'Stores the authenticated agent instance''s config report: mode, + applied/attempted revision, status (applied, rejected, failed or not-applicable; + the server derives pending and unknown), the redacted base and effective configs + (snake_case), the effective digest, plugins (with their agent-library version), + unsafe changes, warnings and the normalized local remote_config block. The + server re-redacts base and effective as a best effort, replaces error, warning + messages, plugin sources, unsafe change values and remote-config strings that + contain a secret with ••••, and stores effective-digest as sent. Long warning + messages and unsafe lists are truncated (truncated=true). A NUL character + anywhere is a 400. Body limit 4 MiB. A 409 means the per-agent instance cap + is reached; back off.' + parameters: + - description: Agent instance ID (UUID) + in: path + name: instanceId + required: true + type: string + - description: Config report + in: body + name: report + required: true + schema: + $ref: '#/definitions/agentconfig.Report' + responses: + "204": + description: No Content + "400": + description: Bad Request + schema: + $ref: '#/definitions/api.Error' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/api.Error' + "403": + description: Forbidden + schema: + $ref: '#/definitions/api.Error' + "409": + description: Conflict + schema: + $ref: '#/definitions/api.Error' + "413": + description: Request Entity Too Large + schema: + $ref: '#/definitions/api.Error' + "415": + description: Unsupported Media Type + schema: + $ref: '#/definitions/api.Error' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/api.Error' + security: + - OAuth2Password: [] + summary: Report this instance's effective configuration + tags: + - Agents /agent/risk-templates/batch: post: consumes: diff --git a/internal/api/handler/agent_config_body.go b/internal/api/handler/agent_config_body.go index a7838b12..6316a58e 100644 --- a/internal/api/handler/agent_config_body.go +++ b/internal/api/handler/agent_config_body.go @@ -1,8 +1,59 @@ package handler +import ( + "errors" + "fmt" + "io" + "mime" + "net/http" + "strings" + + "github.com/compliance-framework/api/internal/api" + "github.com/labstack/echo/v4" +) + // HTTP header names used by the agent-configuration routes. const ( headerETag = "ETag" headerIfMatch = "If-Match" headerIfNoneMatch = "If-None-Match" ) + +// bodyError is a request-body problem with the HTTP status it maps to. +type bodyError struct { + status int + msg string +} + +func (e *bodyError) Error() string { return e.msg } + +// respond writes the error as an api.Error body. +func (e *bodyError) respond(ctx echo.Context) error { + return ctx.JSON(e.status, api.NewError(errors.New(e.msg))) +} + +// readJSONBody reads a JSON request body for the agent-configuration handlers (R13). They do +// not use ctx.Bind, because CustomBinder rejects "application/json; charset=utf-8". A missing +// Content-Type or any application/json media type (with parameters) is accepted; any other +// type is 415. More than limit bytes is 413. An empty body returns nil. +func readJSONBody(ctx echo.Context, limit int64) ([]byte, *bodyError) { + if ct := strings.TrimSpace(ctx.Request().Header.Get(echo.HeaderContentType)); ct != "" { + mediaType, _, err := mime.ParseMediaType(ct) + if err != nil || !strings.EqualFold(mediaType, echo.MIMEApplicationJSON) { + return nil, &bodyError{status: http.StatusUnsupportedMediaType, msg: "request body must be application/json"} + } + } + body := ctx.Request().Body + if body == nil { + return nil, nil + } + data, err := io.ReadAll(http.MaxBytesReader(ctx.Response(), body, limit)) + if err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + return nil, &bodyError{status: http.StatusRequestEntityTooLarge, msg: fmt.Sprintf("request body exceeds %d bytes", limit)} + } + return nil, &bodyError{status: http.StatusBadRequest, msg: "failed to read request body"} + } + return data, nil +} diff --git a/internal/api/handler/agent_config_sync.go b/internal/api/handler/agent_config_sync.go index 5dbb0a00..fdab2816 100644 --- a/internal/api/handler/agent_config_sync.go +++ b/internal/api/handler/agent_config_sync.go @@ -1,10 +1,15 @@ package handler import ( + "bytes" "encoding/json" "errors" + "fmt" "net/http" "regexp" + "slices" + "strings" + "unicode/utf8" "github.com/compliance-framework/api/internal/api" "github.com/compliance-framework/api/internal/api/middleware" @@ -19,6 +24,24 @@ const ( // headerRemoteConfig marks responses from the remote-configuration routes, so an agent can // tell them apart from a proxy's. headerRemoteConfig = "X-CCF-Remote-Config" + + maxReportHostnameLen = 255 + maxReportAgentVersionLen = 64 + maxReportErrorBytes = 8 << 10 + maxReportWarnings = 500 + + // Bounds on the summary columns ListInstances returns for every instance. + maxReportWarningMessageBytes = 1 << 10 + maxReportWarningPathBytes = 1 << 10 + maxReportUnsafe = 200 + maxReportChangePathBytes = 1 << 10 + maxReportChangeValueBytes = 2048 + + // R76: plugins. + maxReportPlugins = 500 + maxReportPluginNameLen = 255 + maxReportPluginSourceLen = 2048 + maxReportPluginLibVersionLen = 64 ) var effectiveDigestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) @@ -35,10 +58,11 @@ func NewAgentConfigSyncHandler(sugar *zap.SugaredLogger, svc *agentcfg.Service) return &AgentConfigSyncHandler{sugar: sugar, svc: svc} } -// Register mounts GET /config on the /agent group. Pass the strict agent JWT middleware and -// the agent:sync guard. +// Register mounts GET /config and PUT /instances/:instanceId/config-report on the /agent +// group. Pass the strict agent JWT middleware and the agent:sync guard. func (h *AgentConfigSyncHandler) Register(g *echo.Group, middlewares ...echo.MiddlewareFunc) { g.GET("/config", h.GetConfig, middlewares...) + g.PUT("/instances/:instanceId/config-report", h.PutReport, middlewares...) } // agentAuthFrom returns the authenticated agent, or nil. The handler requires it even though @@ -122,3 +146,270 @@ func (h *AgentConfigSyncHandler) GetConfig(ctx echo.Context) error { } return ctx.JSON(http.StatusOK, GenericDataResponse[agentconfig.OverlayDocument]{Data: doc}) } + +// PutReport godoc +// +// @Summary Report this instance's effective configuration +// @Description Stores the authenticated agent instance's config report: mode, applied/attempted revision, status (applied, rejected, failed or not-applicable; the server derives pending and unknown), the redacted base and effective configs (snake_case), the effective digest, plugins (with their agent-library version), unsafe changes, warnings and the normalized local remote_config block. The server re-redacts base and effective as a best effort, replaces error, warning messages, plugin sources, unsafe change values and remote-config strings that contain a secret with ••••, and stores effective-digest as sent. Long warning messages and unsafe lists are truncated (truncated=true). A NUL character anywhere is a 400. Body limit 4 MiB. A 409 means the per-agent instance cap is reached; back off. +// @Tags Agents +// @Accept json +// @Param instanceId path string true "Agent instance ID (UUID)" +// @Param report body agentconfig.Report true "Config report" +// @Success 204 "No Content" +// @Failure 400 {object} api.Error +// @Failure 401 {object} api.Error +// @Failure 403 {object} api.Error +// @Failure 409 {object} api.Error +// @Failure 413 {object} api.Error +// @Failure 415 {object} api.Error +// @Failure 500 {object} api.Error +// @Security OAuth2Password +// @Router /agent/instances/{instanceId}/config-report [put] +func (h *AgentConfigSyncHandler) PutReport(ctx echo.Context) error { + auth := agentAuthFrom(ctx) + if auth == nil { + return ctx.JSON(http.StatusUnauthorized, api.NewError(errors.New("agent authentication required"))) + } + setRemoteConfigHeaders(ctx) + agentID := *auth.Agent.ID + + instanceID, err := uuid.Parse(ctx.Param("instanceId")) + if err != nil { + return ctx.JSON(http.StatusBadRequest, api.InvalidUUID()) + } + body, bodyErr := readJSONBody(ctx, agentconfig.MaxReportBytes) + if bodyErr != nil { + return bodyErr.respond(ctx) + } + var report agentconfig.Report + if err := json.Unmarshal(body, &report); err != nil { + return ctx.JSON(http.StatusBadRequest, api.NewError(fmt.Errorf("invalid report body: %w", err))) + } + if err := normalizeReport(&report); err != nil { + return ctx.JSON(http.StatusBadRequest, api.NewError(err)) + } + + // Best-effort server-side re-redaction; the digest is stored exactly as sent (R55). + for _, doc := range []struct { + name string + raw *json.RawMessage + }{{"base", &report.Base}, {"effective", &report.Effective}} { + redacted, changed, err := agentconfig.RedactDocument(*doc.raw) + if err != nil { + return ctx.JSON(http.StatusBadRequest, api.NewError(fmt.Errorf("%s: %w", doc.name, err))) + } + if changed { + h.sugar.Warnw("Agent config report was not fully redacted; re-redacted server-side", + "agentID", agentID, "instanceID", instanceID, "document", doc.name) + } + *doc.raw = redacted + } + if scrubReportText(&report) { + h.sugar.Warnw("Agent config report carried a secret in free text; masked server-side", + "agentID", agentID, "instanceID", instanceID) + } + + var credentialID *uuid.UUID + if auth.Key != nil && auth.Key.ID != nil { + id := *auth.Key.ID + credentialID = &id + } + if err := h.svc.UpsertReport(ctx.Request().Context(), agentID, credentialID, instanceID, report); err != nil { + if errors.Is(err, agentcfg.ErrInstanceLimit) { + return ctx.JSON(http.StatusConflict, api.NewError(agentcfg.ErrInstanceLimit)) + } + h.sugar.Errorw("Failed to store agent config report", "agentID", agentID, "instanceID", instanceID, "error", err) + return ctx.JSON(http.StatusInternalServerError, api.InternalServerError()) + } + return ctx.NoContent(http.StatusNoContent) +} + +// scrubReportText masks the free-text fields of a report that contain a secret by content +// (agentconfig.ScrubSecretText): error, warning messages, plugin sources, unsafe change +// values (a source or env name) and the strings of remote-config. It reports whether +// anything was masked. +func scrubReportText(r *agentconfig.Report) bool { + scrubbed := false + scrub := func(s *string) { + if masked, ok := agentconfig.ScrubSecretText(*s); ok { + *s = masked + scrubbed = true + } + } + if r.Error != nil { + scrub(r.Error) + } + for i := range r.Warnings { + scrub(&r.Warnings[i].Message) + } + for i := range r.Plugins { + scrub(&r.Plugins[i].Source) + } + for i := range r.Unsafe { + scrub(&r.Unsafe[i].Value) + } + if rc := r.RemoteConfig; rc != nil { + scrub(&rc.Mode) + scrub(&rc.PollInterval) + for i := range rc.TrustedSources { + scrub(&rc.TrustedSources[i]) + } + for i := range rc.OverridableConfigFlags { + scrub(&rc.OverridableConfigFlags[i]) + } + } + return scrubbed +} + +// checkReportNUL rejects a NUL character anywhere in a report: Postgres stores neither NUL +// in text columns nor the \u0000 escape in jsonb, so the insert would fail with a 500. +func checkReportNUL(r *agentconfig.Report) error { + for name, v := range map[string]string{ + "hostname": r.Hostname, "agent-version": r.AgentVersion, "effective-digest": r.EffectiveDigest, + } { + if strings.ContainsRune(v, 0) { + return fmt.Errorf("%s must not contain a NUL character", name) + } + } + if r.Error != nil && strings.ContainsRune(*r.Error, 0) { + return errors.New("error must not contain a NUL character") + } + for name, raw := range map[string]json.RawMessage{"base": r.Base, "effective": r.Effective} { + if hasJSONNULEscape(raw) { + return fmt.Errorf("%s must not contain a NUL character", name) + } + } + // The remaining parts are stored as jsonb; encoding/json escapes a NUL as \u0000. + for name, v := range map[string]any{ + "warnings": r.Warnings, "unsafe": r.Unsafe, "plugins": r.Plugins, "remote-config": r.RemoteConfig, + } { + raw, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("%s: %w", name, err) + } + if hasJSONNULEscape(raw) { + return fmt.Errorf("%s must not contain a NUL character", name) + } + } + return nil +} + +// hasJSONNULEscape reports whether raw JSON contains the \u0000 escape (an unescaped +// backslash followed by u0000), i.e. a string that decodes to a NUL character. +func hasJSONNULEscape(raw []byte) bool { + const esc = `\u0000` + for i := 0; ; { + j := bytes.Index(raw[i:], []byte(esc)) + if j < 0 { + return false + } + at := i + j + backslashes := 0 + for k := at - 1; k >= 0 && raw[k] == '\\'; k-- { + backslashes++ + } + if backslashes%2 == 0 { + return true + } + i = at + len(esc) + } +} + +// normalizeReport validates the enums and shapes of a report and applies the length caps +// (truncating, not rejecting, the free-text fields). +func normalizeReport(r *agentconfig.Report) error { + if !slices.Contains(agentconfig.Modes, r.Mode) { + return fmt.Errorf("mode must be one of %s", strings.Join(agentconfig.Modes, ", ")) + } + if !slices.Contains(agentconfig.AgentStatuses, r.Status) { + return fmt.Errorf("status must be one of %s", strings.Join(agentconfig.AgentStatuses, ", ")) + } + if r.Reason != "" && !slices.Contains(agentconfig.Reasons, r.Reason) { + return fmt.Errorf("reason %q is not a known reason", r.Reason) + } + if r.AppliedRevision != nil && *r.AppliedRevision < 0 { + return errors.New("applied-revision must not be negative") + } + if r.AttemptedRevision != nil && *r.AttemptedRevision < 0 { + return errors.New("attempted-revision must not be negative") + } + if !isJSONObject(r.Base) { + return errors.New("base must be a JSON object") + } + if !isJSONObject(r.Effective) { + return errors.New("effective must be a JSON object") + } + if err := checkReportNUL(r); err != nil { + return err + } + if !effectiveDigestPattern.MatchString(r.EffectiveDigest) { + return errors.New("effective-digest must match sha256:<64 lowercase hex>") + } + for i, p := range r.Plugins { + if strings.TrimSpace(p.Name) == "" { + return fmt.Errorf("plugins[%d].name is required", i) + } + } + if len(r.Plugins) > maxReportPlugins { + r.Plugins = r.Plugins[:maxReportPlugins] + r.Truncated = true + } + for i := range r.Plugins { + p := &r.Plugins[i] + p.Name = truncateUTF8(p.Name, maxReportPluginNameLen) + p.Source = truncateUTF8(p.Source, maxReportPluginSourceLen) + p.LibVersion = truncateUTF8(strings.TrimSpace(p.LibVersion), maxReportPluginLibVersionLen) + } + if len(r.Warnings) > maxReportWarnings { + r.Warnings = r.Warnings[:maxReportWarnings] + r.Truncated = true + } + for i := range r.Warnings { + w := &r.Warnings[i] + w.Message = truncateReportField(r, w.Message, maxReportWarningMessageBytes) + w.Path = truncateReportField(r, w.Path, maxReportWarningPathBytes) + } + if len(r.Unsafe) > maxReportUnsafe { + r.Unsafe = r.Unsafe[:maxReportUnsafe] + r.Truncated = true + } + for i := range r.Unsafe { + c := &r.Unsafe[i] + c.Path = truncateReportField(r, c.Path, maxReportChangePathBytes) + c.Value = truncateReportField(r, c.Value, maxReportChangeValueBytes) + } + r.Hostname = truncateUTF8(strings.TrimSpace(r.Hostname), maxReportHostnameLen) + r.AgentVersion = truncateUTF8(strings.TrimSpace(r.AgentVersion), maxReportAgentVersionLen) + if r.Error != nil { + msg := truncateUTF8(*r.Error, maxReportErrorBytes) + r.Error = &msg + } + return nil +} + +// truncateReportField cuts s to n bytes (truncateUTF8) and marks the report truncated when +// it did. +func truncateReportField(r *agentconfig.Report, s string, n int) string { + if len(s) <= n { + return s + } + r.Truncated = true + return truncateUTF8(s, n) +} + +func isJSONObject(raw json.RawMessage) bool { + trimmed := bytes.TrimSpace(raw) + return len(trimmed) > 0 && trimmed[0] == '{' +} + +// truncateUTF8 cuts s to at most n bytes without splitting a rune. +func truncateUTF8(s string, n int) string { + if len(s) <= n { + return s + } + cut := n + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + return s[:cut] +} diff --git a/internal/api/handler/agent_config_sync_integration_test.go b/internal/api/handler/agent_config_sync_integration_test.go index cd151de2..a7638a39 100644 --- a/internal/api/handler/agent_config_sync_integration_test.go +++ b/internal/api/handler/agent_config_sync_integration_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/compliance-framework/api/internal/api" + "github.com/compliance-framework/api/internal/config" "github.com/compliance-framework/api/internal/service/relational" "github.com/compliance-framework/api/internal/service/relational/agentcfg" "github.com/compliance-framework/api/internal/tests" @@ -92,6 +93,19 @@ func (s *AgentConfigSyncIntegrationSuite) getConfig(token, ifNoneMatch string) * return s.do(s.server, http.MethodGet, "/api/agent/config", token, nil, headers) } +func (s *AgentConfigSyncIntegrationSuite) putReport(server *api.Server, token string, instanceID string, body any, headers map[string]string) *httptest.ResponseRecorder { + var raw []byte + switch b := body.(type) { + case []byte: + raw = b + default: + var err error + raw, err = json.Marshal(body) + s.Require().NoError(err) + } + return s.do(server, http.MethodPut, "/api/agent/instances/"+instanceID+"/config-report", token, raw, headers) +} + func (s *AgentConfigSyncIntegrationSuite) createRevision(agentID uuid.UUID, expected int64, overlay string) *relational.AgentConfigRevision { rev, err := s.svc.CreateRevision(context.Background(), agentcfg.CreateRevisionParams{ AgentID: agentID, @@ -115,6 +129,21 @@ func (s *AgentConfigSyncIntegrationSuite) assertRemoteConfigHeaders(rec *httptes s.Equal("no-cache", rec.Header().Get(echo.HeaderCacheControl)) } +func validReportBody() map[string]any { + return map[string]any{ + "hostname": "host-1", + "agent-version": "v0.9.0", + "mode": agentconfig.ModeApplySafe, + "daemon": true, + "applied-revision": 0, + "status": agentconfig.StatusApplied, + "error": nil, + "base": map[string]any{"api": map[string]any{"url": "http://api"}}, + "effective": map[string]any{"api": map[string]any{"url": "http://api"}}, + "effective-digest": syncTestDigest, + } +} + // ---- GET /api/agent/config ---- func (s *AgentConfigSyncIntegrationSuite) TestGetConfigRevisionZero() { @@ -220,6 +249,7 @@ func (s *AgentConfigSyncIntegrationSuite) TestGetConfigAuth() { b := s.newAgent("inactive") s.Require().NoError(s.DB.Exec("UPDATE ccf_agents SET is_active = false WHERE id = ?", *b.agent.ID).Error) s.Equal(http.StatusForbidden, s.getConfig(b.token, "").Code, "inactive agent") + s.Equal(http.StatusForbidden, s.putReport(s.server, b.token, uuid.NewString(), validReportBody(), nil).Code, "inactive agent report") } func (s *AgentConfigSyncIntegrationSuite) TestGetConfigCrossAgentIsolation() { @@ -252,6 +282,331 @@ func (s *AgentConfigSyncIntegrationSuite) TestGetConfigCrossAgentIsolation() { // ---- PUT /api/agent/instances/:instanceId/config-report ---- +func (s *AgentConfigSyncIntegrationSuite) TestPutReportStored() { + a := s.newAgent("reporter") + instanceID := uuid.New() + body := validReportBody() + body["daemon"] = false + body["truncated"] = true + body["mode"] = agentconfig.ModeReport + body["status"] = agentconfig.StatusNotApplicable + body["attempted-revision"] = 0 + body["warnings"] = []map[string]any{{"path": "/plugins/x/schedule", "code": "cron", "message": "bad cron"}} + + rec := s.putReport(s.server, a.token, instanceID.String(), body, nil) + s.Require().Equal(http.StatusNoContent, rec.Code, rec.Body.String()) + s.assertRemoteConfigHeaders(rec) + + row, ok := s.instance(*a.agent.ID, instanceID) + s.Require().True(ok) + s.Equal(agentconfig.ModeReport, row.Mode) + s.Require().NotNil(row.Daemon) + s.False(*row.Daemon) + s.True(row.Truncated) + s.Equal(agentconfig.StatusNotApplicable, row.ReportedStatus) + s.Require().NotNil(row.CredentialID) + s.Equal(*a.key.ID, *row.CredentialID) + s.Require().NotNil(row.EffectiveDigest) + s.Equal(syncTestDigest, *row.EffectiveDigest) + s.Require().NotNil(row.Hostname) + s.Equal("host-1", *row.Hostname) + s.Require().NotNil(row.AppliedRevision) + s.Equal(int64(0), *row.AppliedRevision) + s.NotNil(row.ReportedAt) + s.Nil(row.ApplyError) + s.JSONEq(`{"api":{"url":"http://api"}}`, string(row.BaseConfig)) + s.JSONEq(`[{"path":"/plugins/x/schedule","code":"cron","message":"bad cron"}]`, string(row.Warnings)) + + // A second report updates the same row. + body = validReportBody() + body["status"] = agentconfig.StatusRejected + body["reason"] = agentconfig.ReasonUnsafeChanges + body["error"] = "nope" + rec = s.putReport(s.server, a.token, instanceID.String(), body, nil) + s.Require().Equal(http.StatusNoContent, rec.Code, rec.Body.String()) + row, _ = s.instance(*a.agent.ID, instanceID) + s.Equal(agentconfig.StatusRejected, row.ReportedStatus) + s.Equal(agentconfig.ModeApplySafe, row.Mode) + s.Require().NotNil(row.ApplyReason) + s.Equal(agentconfig.ReasonUnsafeChanges, *row.ApplyReason) + s.Require().NotNil(row.ApplyError) + s.Equal("nope", *row.ApplyError) + var count int64 + s.Require().NoError(s.DB.Model(&relational.AgentInstance{}).Where("agent_id = ?", *a.agent.ID).Count(&count).Error) + s.Equal(int64(1), count) +} + +// TestPutReportIgnoresPolicyBundles: agents built against an earlier revision of this API +// send a policy-bundles inventory. The field is no longer part of the report; it is ignored, +// even when malformed, and the report is stored. +func (s *AgentConfigSyncIntegrationSuite) TestPutReportIgnoresPolicyBundles() { + a := s.newAgent("policy-bundles") + instanceID := uuid.New() + body := validReportBody() + body["policy-bundles"] = []map[string]any{{ + "source": "ghcr.io/vendor/ssh-policies:v1", + "digest": "tree:" + syncTestDigest, + "files": []any{map[string]any{"path": "a.rego", "sha256": "x"}}, + "artifact-digest": "sha256:XYZ", + }} + + rec := s.putReport(s.server, a.token, instanceID.String(), body, nil) + s.Require().Equal(http.StatusNoContent, rec.Code, rec.Body.String()) + + _, ok := s.instance(*a.agent.ID, instanceID) + s.True(ok) +} + +// TestPutReportPlugins: plugins[] (R76) is stored as sent and replaced by the next report; an +// agent that omits it clears it. +func (s *AgentConfigSyncIntegrationSuite) TestPutReportPlugins() { + a := s.newAgent("plugins") + instanceID := uuid.New() + body := validReportBody() + body["plugins"] = []map[string]any{ + {"name": "ssh", "source": "ghcr.io/compliance-framework/plugin-local-ssh:v0.2.0", "lib-version": "v0.1.9"}, + {"name": "local", "source": "/plugins/local"}, + } + + rec := s.putReport(s.server, a.token, instanceID.String(), body, nil) + s.Require().Equal(http.StatusNoContent, rec.Code, rec.Body.String()) + + row, ok := s.instance(*a.agent.ID, instanceID) + s.Require().True(ok) + var plugins []agentconfig.PluginReport + s.Require().NoError(json.Unmarshal(row.Plugins, &plugins)) + s.Equal([]agentconfig.PluginReport{ + {Name: "ssh", Source: "ghcr.io/compliance-framework/plugin-local-ssh:v0.2.0", LibVersion: "v0.1.9"}, + {Name: "local", Source: "/plugins/local"}, + }, plugins) + s.NotContains(string(row.Plugins), `"lib-version":""`, "omitted when unknown") + + // An older agent's report has none: it is cleared. + rec = s.putReport(s.server, a.token, instanceID.String(), validReportBody(), nil) + s.Require().Equal(http.StatusNoContent, rec.Code, rec.Body.String()) + row, ok = s.instance(*a.agent.ID, instanceID) + s.Require().True(ok) + s.Empty(row.Plugins) +} + +func (s *AgentConfigSyncIntegrationSuite) TestPutReportReRedacts() { + a := s.newAgent("redact") + instanceID := uuid.New() + leaky := map[string]any{ + "api": map[string]any{ + "url": "http://api", + "auth": map[string]any{"client_id": "cid", "client_secret": "super-secret"}, + }, + "plugins": map[string]any{ + "x": map[string]any{ + "source": "ghcr.io/compliance-framework/x:v1", + "config": map[string]any{ + "password": "hunter2", + "api_key": "${env:X_API_KEY}", + "region": "eu-west-1", + }, + }, + }, + } + body := validReportBody() + body["base"] = leaky + body["effective"] = leaky + + rec := s.putReport(s.server, a.token, instanceID.String(), body, nil) + s.Require().Equal(http.StatusNoContent, rec.Code, rec.Body.String()) + + row, ok := s.instance(*a.agent.ID, instanceID) + s.Require().True(ok) + for name, raw := range map[string][]byte{"base": row.BaseConfig, "effective": row.EffectiveConfig} { + var doc map[string]any + s.Require().NoError(json.Unmarshal(raw, &doc), name) + auth := doc["api"].(map[string]any)["auth"].(map[string]any) + s.NotContains(auth, "client_secret", name) + s.Equal("cid", auth["client_id"], name) + cfg := doc["plugins"].(map[string]any)["x"].(map[string]any)["config"].(map[string]any) + s.Equal(agentconfig.MaskedValue, cfg["password"], name) + s.Equal("${env:X_API_KEY}", cfg["api_key"], name) + s.Equal("eu-west-1", cfg["region"], name) + s.NotContains(string(raw), "super-secret", name) + s.NotContains(string(raw), "hunter2", name) + } + s.Require().NotNil(row.EffectiveDigest) + s.Equal(syncTestDigest, *row.EffectiveDigest, "digest stored exactly as sent, never recomputed") +} + +// Free-text fields that carry a secret are masked whole before they are stored. +func (s *AgentConfigSyncIntegrationSuite) TestPutReportScrubsFreeText() { + a := s.newAgent("scrub") + instanceID := uuid.New() + body := validReportBody() + body["status"] = agentconfig.StatusFailed + body["reason"] = agentconfig.ReasonInternal + body["error"] = "dial postgres://app:hunter2@db:5432/app: connection refused" + body["warnings"] = []map[string]any{ + {"path": "/plugins/x/config/dsn", "code": agentconfig.FieldCodeInvalidValue, "message": "cannot parse app:hunter2@tcp(db:3306)/app"}, + {"path": "/plugins/x/schedule", "code": agentconfig.FieldCodeCron, "message": "bad cron"}, + } + body["plugins"] = []map[string]any{ + {"name": "x", "source": "https://ci:hunter2@plugins.example.com/x.tar.gz"}, + {"name": "y", "source": "ghcr.io/compliance-framework/y:v1"}, + } + body["remote-config"] = map[string]any{ + "mode": agentconfig.ModeApplySafe, + "trusted_sources": []string{"https://u:hunter2@registry.example.com/*"}, + } + body["unsafe"] = []map[string]any{ + {"path": "/plugins/x/source", "safety": "unsafe", "reason": agentconfig.ChangeReasonUntrustedSource, "value": "oci://u:hunter2@reg/x"}, + } + + rec := s.putReport(s.server, a.token, instanceID.String(), body, nil) + s.Require().Equal(http.StatusNoContent, rec.Code, rec.Body.String()) + + row, ok := s.instance(*a.agent.ID, instanceID) + s.Require().True(ok) + s.Require().NotNil(row.ApplyError) + s.Equal(agentconfig.MaskedValue, *row.ApplyError) + for name, raw := range map[string][]byte{"warnings": row.Warnings, "plugins": row.Plugins, "remote-config": row.RemoteConfig, "unsafe": row.UnsafeChanges} { + s.NotContains(string(raw), "hunter2", name) + } + var warnings []agentconfig.FieldError + s.Require().NoError(json.Unmarshal(row.Warnings, &warnings)) + s.Equal(agentconfig.MaskedValue, warnings[0].Message) + s.Equal("bad cron", warnings[1].Message) + var plugins []agentconfig.PluginReport + s.Require().NoError(json.Unmarshal(row.Plugins, &plugins)) + s.Equal(agentconfig.MaskedValue, plugins[0].Source) + s.Equal("ghcr.io/compliance-framework/y:v1", plugins[1].Source) +} + +func (s *AgentConfigSyncIntegrationSuite) TestPutReportValidation() { + a := s.newAgent("validation") + instanceID := uuid.NewString() + + cases := map[string]func(b map[string]any){ + "status pending": func(b map[string]any) { b["status"] = agentconfig.StatusPending }, + "status unknown": func(b map[string]any) { b["status"] = agentconfig.StatusUnknown }, + "unknown reason": func(b map[string]any) { b["reason"] = "cosmic-rays" }, + "bad mode": func(b map[string]any) { b["mode"] = "apply" }, + "missing mode": func(b map[string]any) { delete(b, "mode") }, + "bad digest": func(b map[string]any) { b["effective-digest"] = "sha256:XYZ" }, + "uppercase digest": func(b map[string]any) { b["effective-digest"] = strings.ToUpper(syncTestDigest) }, + "base not an object": func(b map[string]any) { b["base"] = []any{1, 2} }, + "base null": func(b map[string]any) { b["base"] = nil }, + "effective not an object": func(b map[string]any) { b["effective"] = "x" }, + "negative applied-revision": func(b map[string]any) { b["applied-revision"] = -1 }, + "negative attempted": func(b map[string]any) { b["attempted-revision"] = -3 }, + "wrong type": func(b map[string]any) { b["daemon"] = "yes" }, + "plugin without a name": func(b map[string]any) { + b["plugins"] = []map[string]any{{"source": "ghcr.io/x/p:1", "lib-version": "v0.7.1"}} + }, + "plugins not a list": func(b map[string]any) { b["plugins"] = map[string]any{"ssh": "v0.7.1"} }, + "NUL in hostname": func(b map[string]any) { b["hostname"] = "host\x00" }, + "NUL in error": func(b map[string]any) { b["error"] = "boom\x00" }, + "NUL in base": func(b map[string]any) { b["base"] = map[string]any{"a": "\x00"} }, + "NUL in warning": func(b map[string]any) { + b["warnings"] = []map[string]any{{"path": "/x", "code": "c", "message": "\x00"}} + }, + } + for name, mutate := range cases { + body := validReportBody() + mutate(body) + rec := s.putReport(s.server, a.token, instanceID, body, nil) + s.Equal(http.StatusBadRequest, rec.Code, "%s: %s", name, rec.Body.String()) + } + + rec := s.putReport(s.server, a.token, instanceID, []byte(`{not json`), nil) + s.Equal(http.StatusBadRequest, rec.Code, "malformed JSON") + + rec = s.putReport(s.server, a.token, "not-a-uuid", validReportBody(), nil) + s.Equal(http.StatusBadRequest, rec.Code, "invalid instance id") + + _, ok := s.instance(*a.agent.ID, uuid.MustParse(instanceID)) + s.False(ok, "no rejected report is stored") +} + +func (s *AgentConfigSyncIntegrationSuite) TestPutReportContentTypeAndSize() { + a := s.newAgent("content-type") + raw, err := json.Marshal(validReportBody()) + s.Require().NoError(err) + + rec := s.putReport(s.server, a.token, uuid.NewString(), raw, map[string]string{echo.HeaderContentType: "text/plain"}) + s.Equal(http.StatusUnsupportedMediaType, rec.Code, rec.Body.String()) + + rec = s.putReport(s.server, a.token, uuid.NewString(), raw, map[string]string{echo.HeaderContentType: "application/json; charset=utf-8"}) + s.Equal(http.StatusNoContent, rec.Code, rec.Body.String()) + + // Over 4 MiB: 413 (padding inside a valid JSON document). + body := validReportBody() + body["hostname"] = strings.Repeat("h", agentconfig.MaxReportBytes) + rec = s.putReport(s.server, a.token, uuid.NewString(), body, nil) + s.Equal(http.StatusRequestEntityTooLarge, rec.Code) + + // No token. + rec = s.putReport(s.server, "", uuid.NewString(), raw, nil) + s.Equal(http.StatusUnauthorized, rec.Code) + + userToken, err := s.GetAuthToken() + s.Require().NoError(err) + rec = s.putReport(s.server, *userToken, uuid.NewString(), raw, nil) + s.Equal(http.StatusUnauthorized, rec.Code, "user JWT") +} + +func (s *AgentConfigSyncIntegrationSuite) TestPutReportTruncation() { + a := s.newAgent("truncation") + instanceID := uuid.New() + body := validReportBody() + body["status"] = agentconfig.StatusFailed + body["reason"] = agentconfig.ReasonInternal + body["error"] = strings.Repeat("e", 9000) + warnings := make([]map[string]any, 501) + for i := range warnings { + warnings[i] = map[string]any{"path": fmt.Sprintf("/plugins/p%d", i), "code": "cron", "message": "bad"} + } + body["warnings"] = warnings + body["truncated"] = false + + rec := s.putReport(s.server, a.token, instanceID.String(), body, nil) + s.Require().Equal(http.StatusNoContent, rec.Code, rec.Body.String()) + + row, ok := s.instance(*a.agent.ID, instanceID) + s.Require().True(ok) + s.Require().NotNil(row.ApplyError) + s.LessOrEqual(len(*row.ApplyError), 8192) + s.NotEmpty(*row.ApplyError) + var stored []agentconfig.FieldError + s.Require().NoError(json.Unmarshal(row.Warnings, &stored)) + s.Len(stored, 500) + s.Equal("/plugins/p499", stored[499].Path) + s.True(row.Truncated) +} + +func (s *AgentConfigSyncIntegrationSuite) TestPutReportInstanceCap() { + saved := s.Config.Agents + defer func() { s.Config.Agents = saved }() + cfg := config.DefaultAgentsConfig() + cfg.MaxInstancesPerAgent = 1 + s.Config.Agents = cfg + server := s.buildServer() + + a := s.newAgent("capped") + first := uuid.NewString() + rec := s.putReport(server, a.token, first, validReportBody(), nil) + s.Require().Equal(http.StatusNoContent, rec.Code, rec.Body.String()) + + rec = s.putReport(server, a.token, uuid.NewString(), validReportBody(), nil) + s.Require().Equal(http.StatusConflict, rec.Code, rec.Body.String()) + s.JSONEq(`{"errors":{"body":"instance limit reached"}}`, rec.Body.String()) + + // The existing instance can still report. + rec = s.putReport(server, a.token, first, validReportBody(), nil) + s.Equal(http.StatusNoContent, rec.Code, rec.Body.String()) + + // The cap is per agent. + b := s.newAgent("capped-b") + rec = s.putReport(server, b.token, uuid.NewString(), validReportBody(), nil) + s.Equal(http.StatusNoContent, rec.Code, rec.Body.String()) +} + // ---- POST /api/agent/heartbeat ---- func (s *AgentConfigSyncIntegrationSuite) heartbeat(token string, instanceID uuid.UUID, rev *int64, digest *string) *httptest.ResponseRecorder { @@ -287,6 +642,36 @@ func (s *AgentConfigSyncIntegrationSuite) TestHeartbeatWithDigestRegistersInstan s.Equal(*a.key.ID, *row.CredentialID) } +func (s *AgentConfigSyncIntegrationSuite) TestHeartbeatWithoutDigest() { + a := s.newAgent("hb-no-digest") + instanceID := uuid.New() + + rec := s.heartbeat(a.token, instanceID, nil, nil) + s.Require().Equal(http.StatusCreated, rec.Code, rec.Body.String()) + _, ok := s.instance(*a.agent.ID, instanceID) + s.False(ok, "a heartbeat without a digest never inserts") + + // Once a report exists, a digest-less heartbeat only refreshes last_seen_at. + rec = s.putReport(s.server, a.token, instanceID.String(), validReportBody(), nil) + s.Require().Equal(http.StatusNoContent, rec.Code, rec.Body.String()) + old := time.Now().UTC().Add(-time.Hour) + s.Require().NoError(s.DB.Exec("UPDATE ccf_agent_instances SET last_seen_at = ? WHERE instance_id = ?", old, instanceID).Error) + before, _ := s.instance(*a.agent.ID, instanceID) + + rec = s.heartbeat(a.token, instanceID, nil, nil) + s.Require().Equal(http.StatusCreated, rec.Code, rec.Body.String()) + after, ok := s.instance(*a.agent.ID, instanceID) + s.Require().True(ok) + s.True(after.LastSeenAt.After(before.LastSeenAt.Add(30*time.Minute)), "last_seen_at refreshed") + s.Equal(before.ReportedStatus, after.ReportedStatus) + s.Equal(before.Mode, after.Mode) + s.Equal(before.EffectiveDigest, after.EffectiveDigest) + s.Nil(after.HeartbeatConfigDigest) + s.Nil(after.HeartbeatConfigRevision) + s.Require().NotNil(after.ReportedAt) + s.WithinDuration(*before.ReportedAt, *after.ReportedAt, time.Millisecond) +} + func (s *AgentConfigSyncIntegrationSuite) TestHeartbeatMalformedDigestAndAnonymous() { a := s.newAgent("hb-malformed") instanceID := uuid.New() diff --git a/internal/api/handler/agent_config_sync_test.go b/internal/api/handler/agent_config_sync_test.go new file mode 100644 index 00000000..0684810e --- /dev/null +++ b/internal/api/handler/agent_config_sync_test.go @@ -0,0 +1,358 @@ +package handler + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "testing/iotest" + "unicode/utf8" + + "github.com/compliance-framework/api/pkg/agentconfig" + "github.com/labstack/echo/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testDigest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func validReport() agentconfig.Report { + return agentconfig.Report{ + Mode: agentconfig.ModeApplySafe, + Daemon: true, + Status: agentconfig.StatusApplied, + Base: json.RawMessage(`{}`), + Effective: json.RawMessage(`{"api":{"url":"http://x"}}`), + EffectiveDigest: testDigest, + } +} + +func int64Ptr(v int64) *int64 { return &v } + +func TestNormalizeReport_Valid(t *testing.T) { + for _, mode := range agentconfig.Modes { + for _, status := range agentconfig.AgentStatuses { + r := validReport() + r.Mode = mode + r.Status = status + assert.NoError(t, normalizeReport(&r), "mode=%s status=%s", mode, status) + } + } + for _, reason := range agentconfig.Reasons { + r := validReport() + r.Status = agentconfig.StatusRejected + r.Reason = reason + assert.NoError(t, normalizeReport(&r), "reason=%s", reason) + } + r := validReport() + r.AppliedRevision = int64Ptr(0) + r.AttemptedRevision = int64Ptr(3) + r.Base = json.RawMessage(" \n{\"a\":1}") + assert.NoError(t, normalizeReport(&r)) +} + +func TestNormalizeReport_Rejects(t *testing.T) { + cases := map[string]func(r *agentconfig.Report){ + "empty mode": func(r *agentconfig.Report) { r.Mode = "" }, + "bad mode": func(r *agentconfig.Report) { r.Mode = "apply" }, + "status pending": func(r *agentconfig.Report) { r.Status = agentconfig.StatusPending }, + "status unknown": func(r *agentconfig.Report) { r.Status = agentconfig.StatusUnknown }, + "empty status": func(r *agentconfig.Report) { r.Status = "" }, + "unknown reason": func(r *agentconfig.Report) { r.Reason = "because" }, + "negative applied-revision": func(r *agentconfig.Report) { r.AppliedRevision = int64Ptr(-1) }, + "negative attempted-revision": func(r *agentconfig.Report) { r.AttemptedRevision = int64Ptr(-2) }, + "base missing": func(r *agentconfig.Report) { r.Base = nil }, + "base array": func(r *agentconfig.Report) { r.Base = json.RawMessage(`[1]`) }, + "base null": func(r *agentconfig.Report) { r.Base = json.RawMessage(`null`) }, + "base string": func(r *agentconfig.Report) { r.Base = json.RawMessage(`"x"`) }, + "effective array": func(r *agentconfig.Report) { r.Effective = json.RawMessage(`[]`) }, + "digest empty": func(r *agentconfig.Report) { r.EffectiveDigest = "" }, + "digest no prefix": func(r *agentconfig.Report) { r.EffectiveDigest = strings.TrimPrefix(testDigest, "sha256:") }, + "digest uppercase": func(r *agentconfig.Report) { r.EffectiveDigest = strings.ToUpper(testDigest) }, + "digest short": func(r *agentconfig.Report) { r.EffectiveDigest = testDigest[:len(testDigest)-1] }, + "digest long": func(r *agentconfig.Report) { r.EffectiveDigest = testDigest + "0" }, + "digest non-hex": func(r *agentconfig.Report) { r.EffectiveDigest = testDigest[:len(testDigest)-1] + "g" }, + "plugin without a name": func(r *agentconfig.Report) { + r.Plugins = []agentconfig.PluginReport{{Name: "ssh"}, {Name: " ", LibVersion: "v0.7.1"}} + }, + // NUL: Postgres stores it neither in text nor (as \u0000) in jsonb. + "NUL in hostname": func(r *agentconfig.Report) { r.Hostname = "h\x00st" }, + "NUL in agent-version": func(r *agentconfig.Report) { r.AgentVersion = "v1\x00" }, + "NUL in error": func(r *agentconfig.Report) { e := "boom\x00"; r.Error = &e }, + "NUL in effective-digest": func(r *agentconfig.Report) { r.EffectiveDigest = testDigest[:10] + "\x00" }, + "NUL in warning message": func(r *agentconfig.Report) { + r.Warnings = []agentconfig.FieldError{{Path: "/x", Code: "c", Message: "m\x00"}} + }, + "NUL in plugin source": func(r *agentconfig.Report) { + r.Plugins = []agentconfig.PluginReport{{Name: "ssh", Source: "s\x00"}} + }, + "NUL in plugin name": func(r *agentconfig.Report) { r.Plugins = []agentconfig.PluginReport{{Name: "s\x00sh"}} }, + "NUL in unsafe value": func(r *agentconfig.Report) { + r.Unsafe = []agentconfig.Change{{Path: "/p", Safety: agentconfig.Unsafe, Reason: "r", Value: "\x00"}} + }, + "NUL in remote-config": func(r *agentconfig.Report) { + r.RemoteConfig = &agentconfig.RemoteConfig{Mode: agentconfig.ModeReport, TrustedSources: []string{"a\x00"}} + }, + "NUL escape in base": func(r *agentconfig.Report) { r.Base = json.RawMessage(`{"a":"x\u0000"}`) }, + "NUL escape in effective key": func(r *agentconfig.Report) { r.Effective = json.RawMessage(`{"\u0000":1}`) }, + "NUL escape after an escaped backslash": func(r *agentconfig.Report) { + r.Base = json.RawMessage(`{"a":"\\\u0000"}`) + }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + r := validReport() + mutate(&r) + assert.Error(t, normalizeReport(&r)) + }) + } +} + +func TestNormalizeReport_Truncates(t *testing.T) { + r := validReport() + r.Warnings = make([]agentconfig.FieldError, maxReportWarnings+1) + r.Hostname = " " + strings.Repeat("h", 300) + " " + r.AgentVersion = strings.Repeat("v", 100) + longErr := strings.Repeat("é", maxReportErrorBytes) // 2 bytes each + r.Error = &longErr + require.NoError(t, normalizeReport(&r)) + + assert.Len(t, r.Warnings, maxReportWarnings) + assert.True(t, r.Truncated) + assert.Len(t, r.Hostname, maxReportHostnameLen) + assert.Len(t, r.AgentVersion, maxReportAgentVersionLen) + require.NotNil(t, r.Error) + assert.LessOrEqual(t, len(*r.Error), maxReportErrorBytes) + assert.True(t, utf8.ValidString(*r.Error)) + + // Exactly at the caps: untouched, truncated not forced. + r = validReport() + r.Warnings = make([]agentconfig.FieldError, maxReportWarnings) + r.Hostname = " host " + msg := "short" + r.Error = &msg + require.NoError(t, normalizeReport(&r)) + assert.Len(t, r.Warnings, maxReportWarnings) + assert.False(t, r.Truncated) + assert.Equal(t, "host", r.Hostname) + assert.Equal(t, "short", *r.Error) + + // An agent-set truncated flag is kept. + r = validReport() + r.Truncated = true + require.NoError(t, normalizeReport(&r)) + assert.True(t, r.Truncated) +} + +func TestNormalizeReport_LiteralBackslashU0000IsNotNUL(t *testing.T) { + r := validReport() + r.Base = json.RawMessage(`{"a":"\\u0000"}`) // the 6 characters \u0000, not a NUL + assert.NoError(t, normalizeReport(&r)) +} + +func TestNormalizeReport_CapsSummaryFields(t *testing.T) { + r := validReport() + r.Warnings = []agentconfig.FieldError{{ + Path: strings.Repeat("p", maxReportWarningPathBytes+1), + Code: "c", + Message: strings.Repeat("é", maxReportWarningMessageBytes), // 2 bytes each + }} + r.Unsafe = make([]agentconfig.Change, maxReportUnsafe+1) + r.Unsafe[0] = agentconfig.Change{ + Path: strings.Repeat("p", maxReportChangePathBytes+1), + Safety: agentconfig.Unsafe, + Value: strings.Repeat("v", maxReportChangeValueBytes+1), + } + require.NoError(t, normalizeReport(&r)) + assert.True(t, r.Truncated) + assert.Len(t, r.Warnings[0].Path, maxReportWarningPathBytes) + assert.LessOrEqual(t, len(r.Warnings[0].Message), maxReportWarningMessageBytes) + assert.True(t, utf8.ValidString(r.Warnings[0].Message)) + assert.Len(t, r.Unsafe, maxReportUnsafe) + assert.Len(t, r.Unsafe[0].Path, maxReportChangePathBytes) + assert.Len(t, r.Unsafe[0].Value, maxReportChangeValueBytes) + + // At the caps: untouched. + r = validReport() + r.Warnings = []agentconfig.FieldError{{Path: "/x", Code: "c", Message: strings.Repeat("m", maxReportWarningMessageBytes)}} + r.Unsafe = make([]agentconfig.Change, maxReportUnsafe) + require.NoError(t, normalizeReport(&r)) + assert.False(t, r.Truncated) + assert.Len(t, r.Unsafe, maxReportUnsafe) +} + +func TestScrubReportText(t *testing.T) { + pgURL := "postgres://app:hunter2@db:5432/app" + r := validReport() + errMsg := "dial " + pgURL + ": connection refused" + r.Error = &errMsg + r.Warnings = []agentconfig.FieldError{ + {Path: "/plugins/a/config/x", Code: "c", Message: "cannot reach " + pgURL}, + {Path: "/plugins/b/schedule", Code: "c", Message: "bad cron"}, + } + r.Plugins = []agentconfig.PluginReport{ + {Name: "a", Source: "https://ci:hunter2@plugins.example.com/a.tar.gz"}, + {Name: "b", Source: "ghcr.io/compliance-framework/plugin-b:v1"}, + } + r.RemoteConfig = &agentconfig.RemoteConfig{ + Mode: agentconfig.ModeApplySafe, + TrustedSources: []string{"https://u:p@registry.example.com/*", "ghcr.io/compliance-framework/*"}, + } + r.Unsafe = []agentconfig.Change{ + {Path: "/plugins/a/source", Safety: agentconfig.Unsafe, Reason: agentconfig.ChangeReasonUntrustedSource, Value: "oci://u:hunter2@reg/x"}, + {Path: "/plugins/b/source", Safety: agentconfig.Unsafe, Reason: agentconfig.ChangeReasonUntrustedSource, Value: "ghcr.io/evil/plugin:v1"}, + } + + assert.True(t, scrubReportText(&r)) + assert.Equal(t, agentconfig.MaskedValue, r.Unsafe[0].Value) + assert.Equal(t, "ghcr.io/evil/plugin:v1", r.Unsafe[1].Value) + assert.Equal(t, agentconfig.MaskedValue, *r.Error) + assert.Equal(t, agentconfig.MaskedValue, r.Warnings[0].Message) + assert.Equal(t, "bad cron", r.Warnings[1].Message) + assert.Equal(t, agentconfig.MaskedValue, r.Plugins[0].Source) + assert.Equal(t, "ghcr.io/compliance-framework/plugin-b:v1", r.Plugins[1].Source) + assert.Equal(t, []string{agentconfig.MaskedValue, "ghcr.io/compliance-framework/*"}, r.RemoteConfig.TrustedSources) + assert.Equal(t, agentconfig.ModeApplySafe, r.RemoteConfig.Mode) + + clean := validReport() + assert.False(t, scrubReportText(&clean)) +} + +func TestNormalizeReport_Plugins(t *testing.T) { + r := validReport() + r.Plugins = []agentconfig.PluginReport{ + {Name: "ssh", Source: "ghcr.io/compliance-framework/plugin-local-ssh:v0.2.0", LibVersion: " v0.1.9 "}, + {Name: "local"}, + } + require.NoError(t, normalizeReport(&r)) + assert.Equal(t, []agentconfig.PluginReport{ + {Name: "ssh", Source: "ghcr.io/compliance-framework/plugin-local-ssh:v0.2.0", LibVersion: "v0.1.9"}, + {Name: "local"}, + }, r.Plugins) + assert.False(t, r.Truncated) + + // Over the caps: the plugin list and its free text are cut. + r = validReport() + r.Plugins = make([]agentconfig.PluginReport, maxReportPlugins+1) + for i := range r.Plugins { + r.Plugins[i].Name = "p" + } + r.Plugins[0] = agentconfig.PluginReport{ + Name: strings.Repeat("n", maxReportPluginNameLen+1), + Source: strings.Repeat("s", maxReportPluginSourceLen+1), + LibVersion: strings.Repeat("v", maxReportPluginLibVersionLen+1), + } + require.NoError(t, normalizeReport(&r)) + assert.Len(t, r.Plugins, maxReportPlugins) + assert.Len(t, r.Plugins[0].Name, maxReportPluginNameLen) + assert.Len(t, r.Plugins[0].Source, maxReportPluginSourceLen) + assert.Len(t, r.Plugins[0].LibVersion, maxReportPluginLibVersionLen) + assert.True(t, r.Truncated) +} + +func TestTruncateUTF8(t *testing.T) { + assert.Equal(t, "abc", truncateUTF8("abc", 3)) + assert.Equal(t, "abc", truncateUTF8("abc", 10)) + assert.Equal(t, "ab", truncateUTF8("abc", 2)) + assert.Equal(t, "", truncateUTF8("abc", 0)) + assert.Equal(t, "", truncateUTF8("", 5)) + + // "é" is 2 bytes, "€" 3 bytes, "😀" 4 bytes: never split a rune. + assert.Equal(t, "a", truncateUTF8("aé", 2)) + assert.Equal(t, "aé", truncateUTF8("aé", 3)) + assert.Equal(t, "", truncateUTF8("€", 2)) + assert.Equal(t, "x", truncateUTF8("x😀", 4)) + assert.Equal(t, "x😀", truncateUTF8("x😀y", 5)) + + s := strings.Repeat("日本語", 1000) + for n := 0; n <= 20; n++ { + out := truncateUTF8(s, n) + assert.LessOrEqual(t, len(out), n) + assert.True(t, utf8.ValidString(out), "n=%d", n) + assert.True(t, strings.HasPrefix(s, out)) + assert.Greater(t, len(out), n-3, "cuts at most one partial rune") + } +} + +func TestIsJSONObject(t *testing.T) { + assert.True(t, isJSONObject(json.RawMessage(`{}`))) + assert.True(t, isJSONObject(json.RawMessage(" \t\n{\"a\":1}"))) + assert.False(t, isJSONObject(nil)) + assert.False(t, isJSONObject(json.RawMessage(``))) + assert.False(t, isJSONObject(json.RawMessage(` `))) + assert.False(t, isJSONObject(json.RawMessage(`null`))) + assert.False(t, isJSONObject(json.RawMessage(`[]`))) + assert.False(t, isJSONObject(json.RawMessage(`"{}"`))) + assert.False(t, isJSONObject(json.RawMessage(`1`))) +} + +func jsonBodyContext(contentType, body string) echo.Context { + req := httptest.NewRequest(http.MethodPut, "/", strings.NewReader(body)) + if contentType != "" { + req.Header.Set(echo.HeaderContentType, contentType) + } + return echo.New().NewContext(req, httptest.NewRecorder()) +} + +func TestReadJSONBody(t *testing.T) { + t.Run("accepts application/json", func(t *testing.T) { + data, err := readJSONBody(jsonBodyContext(echo.MIMEApplicationJSON, `{"a":1}`), 1024) + require.Nil(t, err) + assert.Equal(t, `{"a":1}`, string(data)) + }) + t.Run("accepts charset parameter", func(t *testing.T) { + data, err := readJSONBody(jsonBodyContext("application/json; charset=utf-8", `{}`), 1024) + require.Nil(t, err) + assert.Equal(t, `{}`, string(data)) + }) + t.Run("accepts mixed case media type", func(t *testing.T) { + _, err := readJSONBody(jsonBodyContext("Application/JSON", `{}`), 1024) + assert.Nil(t, err) + }) + t.Run("accepts missing content type", func(t *testing.T) { + data, err := readJSONBody(jsonBodyContext("", `{"b":2}`), 1024) + require.Nil(t, err) + assert.Equal(t, `{"b":2}`, string(data)) + }) + t.Run("415 on text/plain", func(t *testing.T) { + _, err := readJSONBody(jsonBodyContext("text/plain", `{}`), 1024) + require.NotNil(t, err) + assert.Equal(t, http.StatusUnsupportedMediaType, err.status) + }) + t.Run("415 on malformed content type", func(t *testing.T) { + _, err := readJSONBody(jsonBodyContext("application/json; =", `{}`), 1024) + require.NotNil(t, err) + assert.Equal(t, http.StatusUnsupportedMediaType, err.status) + }) + t.Run("exactly at the limit", func(t *testing.T) { + data, err := readJSONBody(jsonBodyContext(echo.MIMEApplicationJSON, strings.Repeat("a", 16)), 16) + require.Nil(t, err) + assert.Len(t, data, 16) + }) + t.Run("413 over the limit", func(t *testing.T) { + ctx := jsonBodyContext(echo.MIMEApplicationJSON, strings.Repeat("a", 17)) + _, err := readJSONBody(ctx, 16) + require.NotNil(t, err) + assert.Equal(t, http.StatusRequestEntityTooLarge, err.status) + assert.Equal(t, "request body exceeds 16 bytes", err.msg) + require.NoError(t, err.respond(ctx)) + assert.Equal(t, http.StatusRequestEntityTooLarge, ctx.Response().Status) + }) + t.Run("413 from a MaxBytesReader", func(t *testing.T) { + req := httptest.NewRequest(http.MethodPut, "/", strings.NewReader(strings.Repeat("a", 64))) + rec := httptest.NewRecorder() + req.Body = http.MaxBytesReader(rec, req.Body, 8) + _, err := readJSONBody(echo.New().NewContext(req, rec), 32) + require.NotNil(t, err) + assert.Equal(t, http.StatusRequestEntityTooLarge, err.status) + }) + t.Run("400 on a read error", func(t *testing.T) { + req := httptest.NewRequest(http.MethodPut, "/", iotest.ErrReader(errors.New("boom"))) + _, err := readJSONBody(echo.New().NewContext(req, httptest.NewRecorder()), 32) + require.NotNil(t, err) + assert.Equal(t, http.StatusBadRequest, err.status) + assert.Equal(t, "failed to read request body", err.msg) + }) +} From 2a1a895850490b784749653f3cb4ea418b557f0b Mon Sep 17 00:00:00 2001 From: "ccf-lisa[bot]" <286799724+ccf-lisa[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:01:44 -0300 Subject: [PATCH 2/4] fix(api): mask report secrets before truncating free text normalizeReport cut error, warning messages, unsafe values and plugin sources to their length caps before scrubReportText ran, so a secret straddling a cap no longer matched and its prefix was stored. Masking now runs after the count caps and before the length caps. Co-Authored-By: Claude Opus 5.5 --- internal/api/handler/agent_config_sync.go | 63 ++++++++++--------- .../api/handler/agent_config_sync_test.go | 56 +++++++++++++---- 2 files changed, 79 insertions(+), 40 deletions(-) diff --git a/internal/api/handler/agent_config_sync.go b/internal/api/handler/agent_config_sync.go index fdab2816..5f6c348d 100644 --- a/internal/api/handler/agent_config_sync.go +++ b/internal/api/handler/agent_config_sync.go @@ -185,9 +185,14 @@ func (h *AgentConfigSyncHandler) PutReport(ctx echo.Context) error { if err := json.Unmarshal(body, &report); err != nil { return ctx.JSON(http.StatusBadRequest, api.NewError(fmt.Errorf("invalid report body: %w", err))) } - if err := normalizeReport(&report); err != nil { + scrubbed, err := normalizeReport(&report) + if err != nil { return ctx.JSON(http.StatusBadRequest, api.NewError(err)) } + if scrubbed { + h.sugar.Warnw("Agent config report carried a secret in free text; masked server-side", + "agentID", agentID, "instanceID", instanceID) + } // Best-effort server-side re-redaction; the digest is stored exactly as sent (R55). for _, doc := range []struct { @@ -204,11 +209,6 @@ func (h *AgentConfigSyncHandler) PutReport(ctx echo.Context) error { } *doc.raw = redacted } - if scrubReportText(&report) { - h.sugar.Warnw("Agent config report carried a secret in free text; masked server-side", - "agentID", agentID, "instanceID", instanceID) - } - var credentialID *uuid.UUID if auth.Key != nil && auth.Key.ID != nil { id := *auth.Key.ID @@ -315,64 +315,71 @@ func hasJSONNULEscape(raw []byte) bool { } } -// normalizeReport validates the enums and shapes of a report and applies the length caps -// (truncating, not rejecting, the free-text fields). -func normalizeReport(r *agentconfig.Report) error { +// normalizeReport validates the enums and shapes of a report, applies the count caps, masks +// the free-text fields that contain a secret (scrubReportText) and then applies the length +// caps (truncating, not rejecting). Masking runs before truncation so a secret cut at a cap +// cannot slip past the content checks. scrubbed reports whether anything was masked. +func normalizeReport(r *agentconfig.Report) (scrubbed bool, err error) { if !slices.Contains(agentconfig.Modes, r.Mode) { - return fmt.Errorf("mode must be one of %s", strings.Join(agentconfig.Modes, ", ")) + return false, fmt.Errorf("mode must be one of %s", strings.Join(agentconfig.Modes, ", ")) } if !slices.Contains(agentconfig.AgentStatuses, r.Status) { - return fmt.Errorf("status must be one of %s", strings.Join(agentconfig.AgentStatuses, ", ")) + return false, fmt.Errorf("status must be one of %s", strings.Join(agentconfig.AgentStatuses, ", ")) } if r.Reason != "" && !slices.Contains(agentconfig.Reasons, r.Reason) { - return fmt.Errorf("reason %q is not a known reason", r.Reason) + return false, fmt.Errorf("reason %q is not a known reason", r.Reason) } if r.AppliedRevision != nil && *r.AppliedRevision < 0 { - return errors.New("applied-revision must not be negative") + return false, errors.New("applied-revision must not be negative") } if r.AttemptedRevision != nil && *r.AttemptedRevision < 0 { - return errors.New("attempted-revision must not be negative") + return false, errors.New("attempted-revision must not be negative") } if !isJSONObject(r.Base) { - return errors.New("base must be a JSON object") + return false, errors.New("base must be a JSON object") } if !isJSONObject(r.Effective) { - return errors.New("effective must be a JSON object") + return false, errors.New("effective must be a JSON object") } if err := checkReportNUL(r); err != nil { - return err + return false, err } if !effectiveDigestPattern.MatchString(r.EffectiveDigest) { - return errors.New("effective-digest must match sha256:<64 lowercase hex>") + return false, errors.New("effective-digest must match sha256:<64 lowercase hex>") } for i, p := range r.Plugins { if strings.TrimSpace(p.Name) == "" { - return fmt.Errorf("plugins[%d].name is required", i) + return false, fmt.Errorf("plugins[%d].name is required", i) } } + + // Count caps first, so masking only scans what is kept. if len(r.Plugins) > maxReportPlugins { r.Plugins = r.Plugins[:maxReportPlugins] r.Truncated = true } + if len(r.Warnings) > maxReportWarnings { + r.Warnings = r.Warnings[:maxReportWarnings] + r.Truncated = true + } + if len(r.Unsafe) > maxReportUnsafe { + r.Unsafe = r.Unsafe[:maxReportUnsafe] + r.Truncated = true + } + + scrubbed = scrubReportText(r) + for i := range r.Plugins { p := &r.Plugins[i] p.Name = truncateUTF8(p.Name, maxReportPluginNameLen) p.Source = truncateUTF8(p.Source, maxReportPluginSourceLen) p.LibVersion = truncateUTF8(strings.TrimSpace(p.LibVersion), maxReportPluginLibVersionLen) } - if len(r.Warnings) > maxReportWarnings { - r.Warnings = r.Warnings[:maxReportWarnings] - r.Truncated = true - } for i := range r.Warnings { w := &r.Warnings[i] w.Message = truncateReportField(r, w.Message, maxReportWarningMessageBytes) w.Path = truncateReportField(r, w.Path, maxReportWarningPathBytes) } - if len(r.Unsafe) > maxReportUnsafe { - r.Unsafe = r.Unsafe[:maxReportUnsafe] - r.Truncated = true - } for i := range r.Unsafe { c := &r.Unsafe[i] c.Path = truncateReportField(r, c.Path, maxReportChangePathBytes) @@ -384,7 +391,7 @@ func normalizeReport(r *agentconfig.Report) error { msg := truncateUTF8(*r.Error, maxReportErrorBytes) r.Error = &msg } - return nil + return scrubbed, nil } // truncateReportField cuts s to n bytes (truncateUTF8) and marks the report truncated when diff --git a/internal/api/handler/agent_config_sync_test.go b/internal/api/handler/agent_config_sync_test.go index 0684810e..2a3322f4 100644 --- a/internal/api/handler/agent_config_sync_test.go +++ b/internal/api/handler/agent_config_sync_test.go @@ -31,26 +31,32 @@ func validReport() agentconfig.Report { func int64Ptr(v int64) *int64 { return &v } +// normalizeReportErr is normalizeReport without the scrubbed flag. +func normalizeReportErr(r *agentconfig.Report) error { + _, err := normalizeReport(r) + return err +} + func TestNormalizeReport_Valid(t *testing.T) { for _, mode := range agentconfig.Modes { for _, status := range agentconfig.AgentStatuses { r := validReport() r.Mode = mode r.Status = status - assert.NoError(t, normalizeReport(&r), "mode=%s status=%s", mode, status) + assert.NoError(t, normalizeReportErr(&r), "mode=%s status=%s", mode, status) } } for _, reason := range agentconfig.Reasons { r := validReport() r.Status = agentconfig.StatusRejected r.Reason = reason - assert.NoError(t, normalizeReport(&r), "reason=%s", reason) + assert.NoError(t, normalizeReportErr(&r), "reason=%s", reason) } r := validReport() r.AppliedRevision = int64Ptr(0) r.AttemptedRevision = int64Ptr(3) r.Base = json.RawMessage(" \n{\"a\":1}") - assert.NoError(t, normalizeReport(&r)) + assert.NoError(t, normalizeReportErr(&r)) } func TestNormalizeReport_Rejects(t *testing.T) { @@ -105,7 +111,7 @@ func TestNormalizeReport_Rejects(t *testing.T) { t.Run(name, func(t *testing.T) { r := validReport() mutate(&r) - assert.Error(t, normalizeReport(&r)) + assert.Error(t, normalizeReportErr(&r)) }) } } @@ -117,7 +123,7 @@ func TestNormalizeReport_Truncates(t *testing.T) { r.AgentVersion = strings.Repeat("v", 100) longErr := strings.Repeat("é", maxReportErrorBytes) // 2 bytes each r.Error = &longErr - require.NoError(t, normalizeReport(&r)) + require.NoError(t, normalizeReportErr(&r)) assert.Len(t, r.Warnings, maxReportWarnings) assert.True(t, r.Truncated) @@ -133,7 +139,7 @@ func TestNormalizeReport_Truncates(t *testing.T) { r.Hostname = " host " msg := "short" r.Error = &msg - require.NoError(t, normalizeReport(&r)) + require.NoError(t, normalizeReportErr(&r)) assert.Len(t, r.Warnings, maxReportWarnings) assert.False(t, r.Truncated) assert.Equal(t, "host", r.Hostname) @@ -142,14 +148,14 @@ func TestNormalizeReport_Truncates(t *testing.T) { // An agent-set truncated flag is kept. r = validReport() r.Truncated = true - require.NoError(t, normalizeReport(&r)) + require.NoError(t, normalizeReportErr(&r)) assert.True(t, r.Truncated) } func TestNormalizeReport_LiteralBackslashU0000IsNotNUL(t *testing.T) { r := validReport() r.Base = json.RawMessage(`{"a":"\\u0000"}`) // the 6 characters \u0000, not a NUL - assert.NoError(t, normalizeReport(&r)) + assert.NoError(t, normalizeReportErr(&r)) } func TestNormalizeReport_CapsSummaryFields(t *testing.T) { @@ -165,7 +171,7 @@ func TestNormalizeReport_CapsSummaryFields(t *testing.T) { Safety: agentconfig.Unsafe, Value: strings.Repeat("v", maxReportChangeValueBytes+1), } - require.NoError(t, normalizeReport(&r)) + require.NoError(t, normalizeReportErr(&r)) assert.True(t, r.Truncated) assert.Len(t, r.Warnings[0].Path, maxReportWarningPathBytes) assert.LessOrEqual(t, len(r.Warnings[0].Message), maxReportWarningMessageBytes) @@ -178,7 +184,7 @@ func TestNormalizeReport_CapsSummaryFields(t *testing.T) { r = validReport() r.Warnings = []agentconfig.FieldError{{Path: "/x", Code: "c", Message: strings.Repeat("m", maxReportWarningMessageBytes)}} r.Unsafe = make([]agentconfig.Change, maxReportUnsafe) - require.NoError(t, normalizeReport(&r)) + require.NoError(t, normalizeReportErr(&r)) assert.False(t, r.Truncated) assert.Len(t, r.Unsafe, maxReportUnsafe) } @@ -226,7 +232,7 @@ func TestNormalizeReport_Plugins(t *testing.T) { {Name: "ssh", Source: "ghcr.io/compliance-framework/plugin-local-ssh:v0.2.0", LibVersion: " v0.1.9 "}, {Name: "local"}, } - require.NoError(t, normalizeReport(&r)) + require.NoError(t, normalizeReportErr(&r)) assert.Equal(t, []agentconfig.PluginReport{ {Name: "ssh", Source: "ghcr.io/compliance-framework/plugin-local-ssh:v0.2.0", LibVersion: "v0.1.9"}, {Name: "local"}, @@ -244,7 +250,7 @@ func TestNormalizeReport_Plugins(t *testing.T) { Source: strings.Repeat("s", maxReportPluginSourceLen+1), LibVersion: strings.Repeat("v", maxReportPluginLibVersionLen+1), } - require.NoError(t, normalizeReport(&r)) + require.NoError(t, normalizeReportErr(&r)) assert.Len(t, r.Plugins, maxReportPlugins) assert.Len(t, r.Plugins[0].Name, maxReportPluginNameLen) assert.Len(t, r.Plugins[0].Source, maxReportPluginSourceLen) @@ -356,3 +362,29 @@ func TestReadJSONBody(t *testing.T) { assert.Equal(t, "failed to read request body", err.msg) }) } + +// A secret that straddles a length cap is masked whole: masking runs before truncation, so +// no prefix of it survives. +func TestNormalizeReport_ScrubsBeforeTruncating(t *testing.T) { + secret := "postgres://app:hunter2secretpw@db:5432/app" + r := validReport() + errMsg := strings.Repeat("x", maxReportErrorBytes-20) + " dial " + secret + r.Error = &errMsg + r.Warnings = []agentconfig.FieldError{{Path: "/p", Code: "c", Message: strings.Repeat("y", maxReportWarningMessageBytes-20) + " " + secret}} + r.Unsafe = []agentconfig.Change{{Path: "/plugins/a/source", Safety: agentconfig.Unsafe, Reason: agentconfig.ChangeReasonUntrustedSource, Value: strings.Repeat("z", maxReportChangeValueBytes-20) + " " + secret}} + r.Plugins = []agentconfig.PluginReport{{Name: "a", Source: strings.Repeat("s", maxReportPluginSourceLen-20) + " " + secret}} + + scrubbed, err := normalizeReport(&r) + require.NoError(t, err) + assert.True(t, scrubbed) + assert.Equal(t, agentconfig.MaskedValue, *r.Error) + assert.Equal(t, agentconfig.MaskedValue, r.Warnings[0].Message) + assert.Equal(t, agentconfig.MaskedValue, r.Unsafe[0].Value) + assert.Equal(t, agentconfig.MaskedValue, r.Plugins[0].Source) + assert.False(t, r.Truncated, "masked values are short, so nothing is cut") + + clean := validReport() + scrubbed, err = normalizeReport(&clean) + require.NoError(t, err) + assert.False(t, scrubbed) +} From 7e1cc5af941ec1ce78e87affbc8a407038bd832c Mon Sep 17 00:00:00 2001 From: "ccf-lisa[bot]" <286799724+ccf-lisa[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 08:27:34 -0300 Subject: [PATCH 3/4] fix(api): cap the remote-config block of a config report normalizeReport never bounded remote-config, so a report with 100k trusted_sources (7 MB) was stored with truncated=false, and ListInstances returns that column for every instance. Cap it like the other summary columns, after the scrub: at most 100 trusted_sources and 100 overridable_config_flags, each at most 256 bytes JSON-encoded (an over-long entry is dropped, since a cut glob pattern can widen trust), mode at 32 and poll_interval at 64 bytes. The encoded block stays within 64 KiB even when every character is escaped, and the report is marked truncated. Co-Authored-By: Claude Opus 5.5 --- docs/docs.go | 2 +- docs/swagger.json | 2 +- docs/swagger.yaml | 3 +- ...ig_report_remote_config_regression_test.go | 101 ++++++++++++++++++ internal/api/handler/agent_config_sync.go | 42 +++++++- 5 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 internal/api/handler/agent_config_report_remote_config_regression_test.go diff --git a/docs/docs.go b/docs/docs.go index 1b96cf28..18000aeb 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2703,7 +2703,7 @@ const docTemplate = `{ }, "/agent/instances/{instanceId}/config-report": { "put": { - "description": "Stores the authenticated agent instance's config report: mode, applied/attempted revision, status (applied, rejected, failed or not-applicable; the server derives pending and unknown), the redacted base and effective configs (snake_case), the effective digest, plugins (with their agent-library version), unsafe changes, warnings and the normalized local remote_config block. The server re-redacts base and effective as a best effort, replaces error, warning messages, plugin sources, unsafe change values and remote-config strings that contain a secret with ••••, and stores effective-digest as sent. Long warning messages and unsafe lists are truncated (truncated=true). A NUL character anywhere is a 400. Body limit 4 MiB. A 409 means the per-agent instance cap is reached; back off.", + "description": "Stores the authenticated agent instance's config report: mode, applied/attempted revision, status (applied, rejected, failed or not-applicable; the server derives pending and unknown), the redacted base and effective configs (snake_case), the effective digest, plugins (with their agent-library version), unsafe changes, warnings and the normalized local remote_config block. The server re-redacts base and effective as a best effort, replaces error, warning messages, plugin sources, unsafe change values and remote-config strings that contain a secret with ••••, and stores effective-digest as sent. Long warning messages, unsafe lists and remote-config are truncated (truncated=true); a remote-config list entry over the length cap is dropped, not cut. A NUL character anywhere is a 400. Body limit 4 MiB. A 409 means the per-agent instance cap is reached; back off.", "consumes": [ "application/json" ], diff --git a/docs/swagger.json b/docs/swagger.json index d3fcbf07..ccf3ea61 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2697,7 +2697,7 @@ }, "/agent/instances/{instanceId}/config-report": { "put": { - "description": "Stores the authenticated agent instance's config report: mode, applied/attempted revision, status (applied, rejected, failed or not-applicable; the server derives pending and unknown), the redacted base and effective configs (snake_case), the effective digest, plugins (with their agent-library version), unsafe changes, warnings and the normalized local remote_config block. The server re-redacts base and effective as a best effort, replaces error, warning messages, plugin sources, unsafe change values and remote-config strings that contain a secret with ••••, and stores effective-digest as sent. Long warning messages and unsafe lists are truncated (truncated=true). A NUL character anywhere is a 400. Body limit 4 MiB. A 409 means the per-agent instance cap is reached; back off.", + "description": "Stores the authenticated agent instance's config report: mode, applied/attempted revision, status (applied, rejected, failed or not-applicable; the server derives pending and unknown), the redacted base and effective configs (snake_case), the effective digest, plugins (with their agent-library version), unsafe changes, warnings and the normalized local remote_config block. The server re-redacts base and effective as a best effort, replaces error, warning messages, plugin sources, unsafe change values and remote-config strings that contain a secret with ••••, and stores effective-digest as sent. Long warning messages, unsafe lists and remote-config are truncated (truncated=true); a remote-config list entry over the length cap is dropped, not cut. A NUL character anywhere is a 400. Body limit 4 MiB. A 409 means the per-agent instance cap is reached; back off.", "consumes": [ "application/json" ], diff --git a/docs/swagger.yaml b/docs/swagger.yaml index a1671e9e..ab45a23d 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -14351,7 +14351,8 @@ paths: server re-redacts base and effective as a best effort, replaces error, warning messages, plugin sources, unsafe change values and remote-config strings that contain a secret with ••••, and stores effective-digest as sent. Long warning - messages and unsafe lists are truncated (truncated=true). A NUL character + messages, unsafe lists and remote-config are truncated (truncated=true); a + remote-config list entry over the length cap is dropped, not cut. A NUL character anywhere is a 400. Body limit 4 MiB. A 409 means the per-agent instance cap is reached; back off.' parameters: diff --git a/internal/api/handler/agent_config_report_remote_config_regression_test.go b/internal/api/handler/agent_config_report_remote_config_regression_test.go new file mode 100644 index 00000000..3b571018 --- /dev/null +++ b/internal/api/handler/agent_config_report_remote_config_regression_test.go @@ -0,0 +1,101 @@ +package handler + +import ( + "encoding/json" + "slices" + "strings" + "testing" + + "github.com/compliance-framework/api/pkg/agentconfig" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// maxStoredRemoteConfigBytes is the most remote-config JSON a stored report may keep. The +// block is a summary column returned for every instance by the unpaginated instance list. +const maxStoredRemoteConfigBytes = 64 << 10 + +func regressionReport(rc *agentconfig.RemoteConfig) agentconfig.Report { + return agentconfig.Report{ + Mode: agentconfig.ModeApplySafe, + Status: agentconfig.StatusApplied, + Base: json.RawMessage(`{}`), + Effective: json.RawMessage(`{}`), + EffectiveDigest: "sha256:" + strings.Repeat("a", 64), + RemoteConfig: rc, + } +} + +// Regression (review #480/#476, fp 72f95c2fcf2d): normalizeReport must bound remote-config, +// like warnings, unsafe changes and plugins, and mark the report truncated when it cuts it. +func TestNormalizeReportBoundsRemoteConfig(t *testing.T) { + sources := make([]string, 100_000) + flags := make([]string, 100_000) + for i := range sources { + sources[i] = "ghcr.io/acme/" + strings.Repeat("a", 20) + flags[i] = "plugin:" + strings.Repeat("k", 20) + } + r := regressionReport(&agentconfig.RemoteConfig{ + Mode: agentconfig.ModeApplySafe, + PollInterval: strings.Repeat("p", 500_000), + TrustedSources: sources, + OverridableConfigFlags: flags, + }) + _, err := normalizeReport(&r) + require.NoError(t, err) + raw, err := json.Marshal(r.RemoteConfig) + require.NoError(t, err) + assert.LessOrEqual(t, len(raw), maxStoredRemoteConfigBytes, "remote-config is stored and listed for every instance") + assert.True(t, r.Truncated) +} + +// A normal remote-config block is stored as sent. +func TestNormalizeReportKeepsSmallRemoteConfig(t *testing.T) { + rc := &agentconfig.RemoteConfig{ + Mode: agentconfig.ModeApplySafe, + PollInterval: "60s", + TrustedSources: []string{"ghcr.io/compliance-framework/*"}, + OverridableConfigFlags: []string{"ssh:port"}, + } + r := regressionReport(rc) + _, err := normalizeReport(&r) + require.NoError(t, err) + assert.False(t, r.Truncated) + assert.Equal(t, []string{"ghcr.io/compliance-framework/*"}, r.RemoteConfig.TrustedSources) + assert.Equal(t, []string{"ssh:port"}, r.RemoteConfig.OverridableConfigFlags) + assert.Equal(t, "60s", r.RemoteConfig.PollInterval) +} + +// The bound holds for the worst-case escaping (each '<' encodes to 6 bytes) at every cap, and +// an entry over the length cap is dropped, never cut into a broader pattern. +func TestNormalizeReportRemoteConfigWorstCaseEscaping(t *testing.T) { + escaped := strings.Repeat("<", (maxReportRemoteEntryBytes-2)/6) // just within the cap once encoded + entries := make([]string, maxReportRemoteListEntries) + for i := range entries { + entries[i] = escaped + } + r := regressionReport(&agentconfig.RemoteConfig{ + Mode: strings.Repeat("<", maxReportRemoteModeLen), + PollInterval: strings.Repeat("<", maxReportRemotePollIntervalLen), + TrustedSources: entries, + OverridableConfigFlags: slices.Clone(entries), + }) + _, err := normalizeReport(&r) + require.NoError(t, err) + raw, err := json.Marshal(r.RemoteConfig) + require.NoError(t, err) + assert.LessOrEqual(t, len(raw), maxStoredRemoteConfigBytes) + assert.False(t, r.Truncated, "every entry is within the caps") + assert.Len(t, r.RemoteConfig.TrustedSources, maxReportRemoteListEntries) + + long := "ghcr.io/acme/*/" + strings.Repeat("x", maxReportRemoteEntryBytes) + r = regressionReport(&agentconfig.RemoteConfig{ + TrustedSources: []string{"ghcr.io/ok/*", long}, + OverridableConfigFlags: []string{long, "ssh:port"}, + }) + _, err = normalizeReport(&r) + require.NoError(t, err) + assert.True(t, r.Truncated) + assert.Equal(t, []string{"ghcr.io/ok/*"}, r.RemoteConfig.TrustedSources) + assert.Equal(t, []string{"ssh:port"}, r.RemoteConfig.OverridableConfigFlags) +} diff --git a/internal/api/handler/agent_config_sync.go b/internal/api/handler/agent_config_sync.go index 5f6c348d..e3d5a59a 100644 --- a/internal/api/handler/agent_config_sync.go +++ b/internal/api/handler/agent_config_sync.go @@ -42,6 +42,15 @@ const ( maxReportPluginNameLen = 255 maxReportPluginSourceLen = 2048 maxReportPluginLibVersionLen = 64 + + // remote-config, a summary column too. Its JSON stays within 64 KiB: two lists of at + // most maxReportRemoteListEntries entries, each at most maxReportRemoteEntryBytes once + // JSON-encoded (2 × 100 × 257 bytes with the commas), plus mode and poll_interval (at + // most 6× their byte caps once escaped) and the keys. + maxReportRemoteListEntries = 100 + maxReportRemoteEntryBytes = 256 // JSON-encoded, quotes included + maxReportRemoteModeLen = 32 + maxReportRemotePollIntervalLen = 64 ) var effectiveDigestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) @@ -150,7 +159,7 @@ func (h *AgentConfigSyncHandler) GetConfig(ctx echo.Context) error { // PutReport godoc // // @Summary Report this instance's effective configuration -// @Description Stores the authenticated agent instance's config report: mode, applied/attempted revision, status (applied, rejected, failed or not-applicable; the server derives pending and unknown), the redacted base and effective configs (snake_case), the effective digest, plugins (with their agent-library version), unsafe changes, warnings and the normalized local remote_config block. The server re-redacts base and effective as a best effort, replaces error, warning messages, plugin sources, unsafe change values and remote-config strings that contain a secret with ••••, and stores effective-digest as sent. Long warning messages and unsafe lists are truncated (truncated=true). A NUL character anywhere is a 400. Body limit 4 MiB. A 409 means the per-agent instance cap is reached; back off. +// @Description Stores the authenticated agent instance's config report: mode, applied/attempted revision, status (applied, rejected, failed or not-applicable; the server derives pending and unknown), the redacted base and effective configs (snake_case), the effective digest, plugins (with their agent-library version), unsafe changes, warnings and the normalized local remote_config block. The server re-redacts base and effective as a best effort, replaces error, warning messages, plugin sources, unsafe change values and remote-config strings that contain a secret with ••••, and stores effective-digest as sent. Long warning messages, unsafe lists and remote-config are truncated (truncated=true); a remote-config list entry over the length cap is dropped, not cut. A NUL character anywhere is a 400. Body limit 4 MiB. A 409 means the per-agent instance cap is reached; back off. // @Tags Agents // @Accept json // @Param instanceId path string true "Agent instance ID (UUID)" @@ -366,6 +375,16 @@ func normalizeReport(r *agentconfig.Report) (scrubbed bool, err error) { r.Unsafe = r.Unsafe[:maxReportUnsafe] r.Truncated = true } + if rc := r.RemoteConfig; rc != nil { + if len(rc.TrustedSources) > maxReportRemoteListEntries { + rc.TrustedSources = rc.TrustedSources[:maxReportRemoteListEntries] + r.Truncated = true + } + if len(rc.OverridableConfigFlags) > maxReportRemoteListEntries { + rc.OverridableConfigFlags = rc.OverridableConfigFlags[:maxReportRemoteListEntries] + r.Truncated = true + } + } scrubbed = scrubReportText(r) @@ -385,6 +404,12 @@ func normalizeReport(r *agentconfig.Report) (scrubbed bool, err error) { c.Path = truncateReportField(r, c.Path, maxReportChangePathBytes) c.Value = truncateReportField(r, c.Value, maxReportChangeValueBytes) } + if rc := r.RemoteConfig; rc != nil { + rc.Mode = truncateReportField(r, rc.Mode, maxReportRemoteModeLen) + rc.PollInterval = truncateReportField(r, rc.PollInterval, maxReportRemotePollIntervalLen) + rc.TrustedSources = dropLongRemoteEntries(r, rc.TrustedSources) + rc.OverridableConfigFlags = dropLongRemoteEntries(r, rc.OverridableConfigFlags) + } r.Hostname = truncateUTF8(strings.TrimSpace(r.Hostname), maxReportHostnameLen) r.AgentVersion = truncateUTF8(strings.TrimSpace(r.AgentVersion), maxReportAgentVersionLen) if r.Error != nil { @@ -404,6 +429,21 @@ func truncateReportField(r *agentconfig.Report, s string, n int) string { return truncateUTF8(s, n) } +// dropLongRemoteEntries removes the trusted_sources or overridable_config_flags entries +// longer than maxReportRemoteEntryBytes once JSON-encoded, and marks the report truncated +// when it removed any. An entry is dropped, not cut: these are path.Match patterns, and a +// cut pattern can match more than the host trusts (".../*/x" cut to ".../*"). +func dropLongRemoteEntries(r *agentconfig.Report, entries []string) []string { + return slices.DeleteFunc(entries, func(e string) bool { + encoded, err := json.Marshal(e) + if err == nil && len(encoded) <= maxReportRemoteEntryBytes { + return false + } + r.Truncated = true + return true + }) +} + func isJSONObject(raw json.RawMessage) bool { trimmed := bytes.TrimSpace(raw) return len(trimmed) > 0 && trimmed[0] == '{' From 918d46ae1c072bd92cf88b89402d77a4a130e9b2 Mon Sep 17 00:00:00 2001 From: "ccf-lisa[bot]" <286799724+ccf-lisa[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:44:09 -0300 Subject: [PATCH 4/4] fix(api): bound a config report's summary after JSON escaping warnings[].code, unsafe[].safety and unsafe[].reason had no length cap, and the other caps are on raw bytes, which JSON escaping can grow sixfold ('<', '&' and control characters encode as \u00XX). One report could make its instance's listed summary about 21 MB, and a page of 25 instances about 500 MiB. normalizeReport now cuts code, safety and reason to 64 bytes (cut, not rejected: a newer agent may send values this API does not know) and keeps the summary fields (hostname, agent-version, error, warnings, unsafe, plugins, remote-config) within 3 MiB as encoding/json encodes them with HTML escaping, the way the instance list does. Over that budget it cuts the error text to 8 KiB encoded and drops the last entry of the largest list until the report fits, and marks the report truncated. A plain-text report at every cap (about 2.9 MB) fits, so it is stored unchanged. Co-Authored-By: Claude Opus 5.5 --- ...nt_config_report_budget_regression_test.go | 159 ++++++++++++++++++ internal/api/handler/agent_config_sync.go | 151 ++++++++++++++++- .../service/relational/agentcfg/service.go | 13 +- 3 files changed, 314 insertions(+), 9 deletions(-) create mode 100644 internal/api/handler/agent_config_report_budget_regression_test.go diff --git a/internal/api/handler/agent_config_report_budget_regression_test.go b/internal/api/handler/agent_config_report_budget_regression_test.go new file mode 100644 index 00000000..3ee238e1 --- /dev/null +++ b/internal/api/handler/agent_config_report_budget_regression_test.go @@ -0,0 +1,159 @@ +package handler + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/compliance-framework/api/pkg/agentconfig" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Regression (review #476/#480, fp 5d771b4b3d43): a stored report's summary fields, which the +// instance list returns for every instance on a page, stay within +// maxReportSummaryEncodedBytes once JSON-encoded with HTML escaping, whatever their content. + +// summaryEncodedLen encodes a report's summary fields the way the instance list does +// (encoding/json, HTML escaping on), keys included. +func summaryEncodedLen(t *testing.T, r agentconfig.Report) int { + t.Helper() + raw, err := json.Marshal(struct { + Hostname string `json:"hostname"` + AgentVersion string `json:"agent-version"` + Error *string `json:"error"` + Warnings []agentconfig.FieldError `json:"warnings"` + Unsafe []agentconfig.Change `json:"unsafe"` + Plugins []agentconfig.PluginReport `json:"plugins"` + RemoteConfig *agentconfig.RemoteConfig `json:"remote-config,omitempty"` + }{r.Hostname, r.AgentVersion, r.Error, r.Warnings, r.Unsafe, r.Plugins, r.RemoteConfig}) + require.NoError(t, err) + return len(raw) +} + +// summaryKeysOverhead is what summaryEncodedLen adds to the budgeted fields: the object's +// braces, keys and separators. +const summaryKeysOverhead = 128 + +// capsReport fills every capped summary field of a report with fill: exactly at each count +// and byte cap when over is 0, over them otherwise (one more entry, 16 more bytes). +func capsReport(fill string, over int) agentconfig.Report { + text := func(n int) string { return strings.Repeat(fill, (n+over*16)/len(fill)) } + count := func(n int) int { return n + over } + r := validReport() + r.Hostname = text(maxReportHostnameLen) + r.AgentVersion = text(maxReportAgentVersionLen) + errText := text(maxReportErrorBytes) + r.Error = &errText + for range count(maxReportWarnings) { + r.Warnings = append(r.Warnings, agentconfig.FieldError{ + Path: text(maxReportWarningPathBytes), Code: text(maxReportWarningCodeBytes), Message: text(maxReportWarningMessageBytes), + }) + } + for range count(maxReportUnsafe) { + r.Unsafe = append(r.Unsafe, agentconfig.Change{ + Path: text(maxReportChangePathBytes), Safety: agentconfig.Safety(text(maxReportChangeSafetyBytes)), + Reason: text(maxReportChangeReasonBytes), Value: text(maxReportChangeValueBytes), + }) + } + for range count(maxReportPlugins) { + r.Plugins = append(r.Plugins, agentconfig.PluginReport{ + Name: text(maxReportPluginNameLen), Source: text(maxReportPluginSourceLen), LibVersion: text(maxReportPluginLibVersionLen), + }) + } + // Remote-config entries are capped on their encoded size already; fill them to that cap. + entry, _ := json.Marshal(strings.Repeat(fill, 1)) + perFill := len(entry) - 2 + rc := &agentconfig.RemoteConfig{ + Mode: text(maxReportRemoteModeLen), + PollInterval: text(maxReportRemotePollIntervalLen), + } + for range count(maxReportRemoteListEntries) { + e := strings.Repeat(fill, (maxReportRemoteEntryBytes-2)/perFill) + rc.TrustedSources = append(rc.TrustedSources, e) + rc.OverridableConfigFlags = append(rc.OverridableConfigFlags, e) + } + r.RemoteConfig = rc + return r +} + +// The fields that had no length cap (warnings[].code, unsafe[].safety, unsafe[].reason) are +// cut, not rejected: a newer agent may send codes or reasons this API does not know. +func TestNormalizeReportCapsCodeSafetyAndReason(t *testing.T) { + r := validReport() + r.Warnings = []agentconfig.FieldError{ + {Path: "/a", Code: strings.Repeat("<", 3_500_000), Message: "m"}, + {Path: "/b", Code: "a-code-from-a-newer-agent", Message: "m"}, + } + r.Unsafe = []agentconfig.Change{ + {Path: "/c", Safety: agentconfig.Safety(strings.Repeat("s", 1000)), Reason: strings.Repeat("r", 1000)}, + {Path: "/d", Safety: "a-newer-safety", Reason: "a-newer-reason"}, + } + require.NoError(t, normalizeReportErr(&r)) + assert.True(t, r.Truncated) + assert.Equal(t, strings.Repeat("<", maxReportWarningCodeBytes), r.Warnings[0].Code) + assert.Equal(t, "a-code-from-a-newer-agent", r.Warnings[1].Code, "unknown codes are kept") + assert.Len(t, string(r.Unsafe[0].Safety), maxReportChangeSafetyBytes) + assert.Len(t, r.Unsafe[0].Reason, maxReportChangeReasonBytes) + assert.Equal(t, agentconfig.Safety("a-newer-safety"), r.Unsafe[1].Safety) + assert.Equal(t, "a-newer-reason", r.Unsafe[1].Reason) + assert.Len(t, r.Warnings, 2) + assert.Less(t, summaryEncodedLen(t, r), 4<<10) +} + +// Content that JSON escapes (each '<', '&' or control character encodes to six bytes) is cut +// to the budget: trailing list entries are dropped and the error text is cut. +func TestNormalizeReportSummaryBudgetAfterEscaping(t *testing.T) { + for _, fill := range []string{"<", "&", "\x01", "<&\x1f"} { + for _, over := range []int{0, 1} { + r := capsReport(fill, over) + raw := capsReport(fill, over) + require.NoError(t, normalizeReportErr(&r)) + assert.True(t, r.Truncated, "%q over=%d", fill, over) + size := summaryEncodedLen(t, r) + assert.LessOrEqual(t, size, maxReportSummaryEncodedBytes+summaryKeysOverhead, "%q over=%d", fill, over) + assert.Greater(t, size, maxReportSummaryEncodedBytes*9/10, "%q over=%d: only what is needed is dropped", fill, over) + + errLen, err := encodedLen(*r.Error) + require.NoError(t, err) + assert.LessOrEqual(t, errLen, maxReportErrorEncodedBytes) + assert.True(t, strings.HasPrefix(*raw.Error, *r.Error), "the error text is cut, not replaced") + assert.NotEmpty(t, r.Warnings) + assert.NotEmpty(t, r.Unsafe) + assert.NotEmpty(t, r.Plugins) + // The entries kept are the first ones, as the byte caps left them. + assert.Equal(t, truncateUTF8(raw.Warnings[0].Message, maxReportWarningMessageBytes), r.Warnings[0].Message) + assert.Equal(t, truncateUTF8(raw.Plugins[0].Source, maxReportPluginSourceLen), r.Plugins[0].Source) + } + } +} + +// A plain-text report at every cap fits the budget, so the budget leaves it unchanged. +func TestNormalizeReportPlainTextAtCapsIsUnchanged(t *testing.T) { + r := capsReport("x", 0) + want := capsReport("x", 0) + require.NoError(t, normalizeReportErr(&r)) + assert.False(t, r.Truncated) + assert.Equal(t, want, r) + assert.LessOrEqual(t, summaryEncodedLen(t, r), maxReportSummaryEncodedBytes) +} + +// A typical report is stored as sent. +func TestNormalizeReportTypicalReportIsUnchanged(t *testing.T) { + build := func() agentconfig.Report { + r := validReport() + errText := `plugin "ssh": dial tcp 10.0.0.1:22: i/o timeout` + "\n" + `` + r.Error = &errText + r.Hostname = "host-1.example.com" + r.AgentVersion = "v1.4.0" + r.Warnings = []agentconfig.FieldError{{Path: "/plugins/ssh/foo", Code: agentconfig.FieldCodeUnknownField, Message: `unknown field "foo"`}} + r.Unsafe = []agentconfig.Change{{Path: "/plugins/ssh/source", Safety: agentconfig.Unsafe, Reason: agentconfig.ChangeReasonUntrustedSource, Value: "ghcr.io/x/ssh:v2"}} + r.Plugins = []agentconfig.PluginReport{{Name: "ssh", Source: "ghcr.io/x/ssh:v1", LibVersion: "v0.7.1"}} + r.RemoteConfig = &agentconfig.RemoteConfig{Mode: agentconfig.ModeApplySafe, PollInterval: "60s", TrustedSources: []string{"ghcr.io/x/*"}, OverridableConfigFlags: []string{}} + return r + } + r := build() + require.NoError(t, normalizeReportErr(&r)) + assert.False(t, r.Truncated) + assert.Equal(t, build(), r) +} diff --git a/internal/api/handler/agent_config_sync.go b/internal/api/handler/agent_config_sync.go index e3d5a59a..81ef8991 100644 --- a/internal/api/handler/agent_config_sync.go +++ b/internal/api/handler/agent_config_sync.go @@ -33,9 +33,24 @@ const ( // Bounds on the summary columns ListInstances returns for every instance. maxReportWarningMessageBytes = 1 << 10 maxReportWarningPathBytes = 1 << 10 + maxReportWarningCodeBytes = 64 // codes are an open set: a newer agent may send ones this API does not know maxReportUnsafe = 200 maxReportChangePathBytes = 1 << 10 maxReportChangeValueBytes = 2048 + maxReportChangeSafetyBytes = 64 // safety and reason are cut, not rejected, for the same reason + maxReportChangeReasonBytes = 64 + + // maxReportSummaryEncodedBytes bounds the summary fields of one stored report (hostname, + // agent-version, error, warnings, unsafe, plugins, remote-config) as the instance list + // encodes them: encoding/json with HTML escaping on, as echo's DefaultJSONSerializer does. + // The byte caps above are on raw text, which escaping can grow up to six times (`<`, `&` + // and control characters become \u00XX), so normalizeReport also enforces this budget by + // dropping trailing list entries (applyReportSummaryBudget). A plain-text report at every + // cap encodes to about 2.9 MB, so it is never cut. + maxReportSummaryEncodedBytes = 3 << 20 + // maxReportErrorEncodedBytes is what the error text keeps once the summary budget is + // exceeded: maxReportErrorBytes of text without escapes. + maxReportErrorEncodedBytes = maxReportErrorBytes + 2 // R76: plugins. maxReportPlugins = 500 @@ -325,9 +340,10 @@ func hasJSONNULEscape(raw []byte) bool { } // normalizeReport validates the enums and shapes of a report, applies the count caps, masks -// the free-text fields that contain a secret (scrubReportText) and then applies the length -// caps (truncating, not rejecting). Masking runs before truncation so a secret cut at a cap -// cannot slip past the content checks. scrubbed reports whether anything was masked. +// the free-text fields that contain a secret (scrubReportText), then applies the length caps +// and the encoded summary budget (applyReportSummaryBudget), truncating, not rejecting. +// Masking runs before truncation so a secret cut at a cap cannot slip past the content +// checks. scrubbed reports whether anything was masked. func normalizeReport(r *agentconfig.Report) (scrubbed bool, err error) { if !slices.Contains(agentconfig.Modes, r.Mode) { return false, fmt.Errorf("mode must be one of %s", strings.Join(agentconfig.Modes, ", ")) @@ -398,11 +414,14 @@ func normalizeReport(r *agentconfig.Report) (scrubbed bool, err error) { w := &r.Warnings[i] w.Message = truncateReportField(r, w.Message, maxReportWarningMessageBytes) w.Path = truncateReportField(r, w.Path, maxReportWarningPathBytes) + w.Code = truncateReportField(r, w.Code, maxReportWarningCodeBytes) } for i := range r.Unsafe { c := &r.Unsafe[i] c.Path = truncateReportField(r, c.Path, maxReportChangePathBytes) c.Value = truncateReportField(r, c.Value, maxReportChangeValueBytes) + c.Safety = agentconfig.Safety(truncateReportField(r, string(c.Safety), maxReportChangeSafetyBytes)) + c.Reason = truncateReportField(r, c.Reason, maxReportChangeReasonBytes) } if rc := r.RemoteConfig; rc != nil { rc.Mode = truncateReportField(r, rc.Mode, maxReportRemoteModeLen) @@ -416,9 +435,135 @@ func normalizeReport(r *agentconfig.Report) (scrubbed bool, err error) { msg := truncateUTF8(*r.Error, maxReportErrorBytes) r.Error = &msg } + if err := applyReportSummaryBudget(r); err != nil { + return false, err + } return scrubbed, nil } +// encodedLen is the length of v encoded as the instance list encodes it (encoding/json with +// HTML escaping, which json.Marshal applies). +func encodedLen(v any) (int, error) { + raw, err := json.Marshal(v) + return len(raw), err +} + +// encodedEntryLens returns the encoded length of each entry of a list. +func encodedEntryLens[T any](entries []T) ([]int, error) { + lens := make([]int, len(entries)) + for i := range entries { + n, err := encodedLen(entries[i]) + if err != nil { + return nil, err + } + lens[i] = n + } + return lens, nil +} + +// applyReportSummaryBudget keeps the encoded summary fields of a normalized report within +// maxReportSummaryEncodedBytes. Within budget, the report is left unchanged. Over it, the +// report is marked truncated, the error text is cut to maxReportErrorEncodedBytes encoded, +// and the last entry of the largest list (warnings, unsafe or plugins) is dropped until the +// report fits. The other fields are small once capped (remote-config within 64 KiB encoded, +// the hostname and version within six times their byte caps), so dropping entries always +// reaches the budget. +func applyReportSummaryBudget(r *agentconfig.Report) error { + fixed := 0 + for _, v := range []any{r.Hostname, r.AgentVersion, r.RemoteConfig} { + n, err := encodedLen(v) + if err != nil { + return err + } + fixed += n + } + errLen := 0 + if r.Error != nil { + n, err := encodedLen(*r.Error) + if err != nil { + return err + } + errLen = n + } + // Per list: the encoded length of each entry, how many are kept, and the kept entries' + // encoded length as a JSON array. + type list struct { + entries []int + kept int + size int + } + newList := func(entries []int) list { + l := list{entries: entries, kept: len(entries), size: 2 + max(0, len(entries)-1)} + for _, n := range entries { + l.size += n + } + return l + } + warnings, err := encodedEntryLens(r.Warnings) + if err != nil { + return err + } + unsafe, err := encodedEntryLens(r.Unsafe) + if err != nil { + return err + } + plugins, err := encodedEntryLens(r.Plugins) + if err != nil { + return err + } + lists := [3]list{newList(warnings), newList(unsafe), newList(plugins)} + total := fixed + errLen + lists[0].size + lists[1].size + lists[2].size + if total <= maxReportSummaryEncodedBytes { + return nil + } + r.Truncated = true + if r.Error != nil && errLen > maxReportErrorEncodedBytes { + msg, n, err := truncateEncoded(*r.Error, maxReportErrorEncodedBytes) + if err != nil { + return err + } + r.Error = &msg + total -= errLen - n + } + for total > maxReportSummaryEncodedBytes { + largest := &lists[0] + for i := range lists[1:] { + if lists[i+1].size > largest.size { + largest = &lists[i+1] + } + } + if largest.kept == 0 { + break // unreachable: the fixed fields are far below the budget + } + largest.kept-- + dropped := largest.entries[largest.kept] + if largest.kept > 0 { + dropped++ // its comma + } + largest.size -= dropped + total -= dropped + } + r.Warnings = r.Warnings[:lists[0].kept] + r.Unsafe = r.Unsafe[:lists[1].kept] + r.Plugins = r.Plugins[:lists[2].kept] + return nil +} + +// truncateEncoded cuts s to a rune-boundary prefix whose JSON encoding is at most limit bytes +// (at most a few bytes shorter than the longest such prefix) and returns it with its encoded +// length. +func truncateEncoded(s string, limit int) (string, int, error) { + cut := truncateUTF8(s, limit-2) // quotes included, each byte encodes to at least one byte + for { + n, err := encodedLen(cut) + if err != nil || n <= limit { + return cut, n, err + } + // A byte encodes to at most six, so drop at least a sixth of the overshoot. + cut = truncateUTF8(cut, len(cut)-max(1, (n-limit+5)/6)) + } +} + // truncateReportField cuts s to n bytes (truncateUTF8) and marks the report truncated when // it did. func truncateReportField(r *agentconfig.Report, s string, n int) string { diff --git a/internal/service/relational/agentcfg/service.go b/internal/service/relational/agentcfg/service.go index 739ee973..c08f3f14 100644 --- a/internal/service/relational/agentcfg/service.go +++ b/internal/service/relational/agentcfg/service.go @@ -540,12 +540,13 @@ var summaryColumns = []string{ } // InstancesPageLimit is the default and the maximum page size of ListInstances. A listed -// instance's summary columns are bounded only by the report handler (normalizeReport): about -// 3 MiB of text per instance in the worst case (warnings ~1 MiB, plugins ~1.2 MiB, unsafe -// changes ~0.6 MiB, remote-config 64 KiB, error 8 KiB), and never more than one report body -// (agentconfig.MaxReportBytes, 4 MiB). So a page holds about 75 MiB of summary text at most, -// whatever the agent's instance count (up to MaxInstancesPerAgent non-prunable instances plus -// the prune-eligible ones PruneInstances has not deleted yet). +// instance's summary columns are bounded only by the report handler (normalizeReport): every +// free-text field has a byte cap, and the summary fields as a whole are cut to 3 MiB once +// JSON-encoded with HTML escaping, as the instance list encodes them +// (maxReportSummaryEncodedBytes; a plain-text report at every cap is about 2.9 MB and is never +// cut). So a page encodes to about 75 MiB at most, whatever the agent's instance count (up to +// MaxInstancesPerAgent non-prunable instances plus the prune-eligible ones PruneInstances has +// not deleted yet) and whatever the reports contain. const InstancesPageLimit = 25 // ListInstances returns one page of an agent's instances, most recently seen first