feat(agent): pull and apply remote configuration overlays (apply_safe/apply_all)

cmd/agent.go

@@ -239,7 +239,7 @@ with plugins to ensure continuous compliance.`,
 	agentCmd.Flags().StringP("config", "c", "", "Location of config file")
 	agentCmd.MarkFlagRequired("config")
 
-	agentCmd.Flags().String("state-dir", "", "Directory for this instance's state (instance ID); overrides CCF_STATE_DIR. Default: .compliance-framework/state/<hash of the config path>")
+	agentCmd.Flags().String("state-dir", "", "Directory for this instance's state (instance ID, remote config cache); overrides CCF_STATE_DIR. Default: .compliance-framework/state/<hash of the config path>")
 	agentCmd.Flags().String("instance-id", "", "Pin this instance's UUID (not persisted); overrides CCF_INSTANCE_ID")
 
 	return agentCmd
@@ -1450,6 +1450,7 @@ func (ar *AgentRunner) runAllPlugins(ctx context.Context) error {
 			resultsHelper := runner.NewApiHelper(logger, client, labels, pluginName,
 				runner.WithPolicyPaths(policyPaths),
 				runner.WithSources(sourceOf(pluginConfig.Source, source), policySources),
+				runner.WithEvidenceProps(configRevisionProps(config)...),
 			)
 
 			policyBehaviorProto := policyBehaviorToProto(pluginConfig.PolicyBehavior)
@@ -1594,6 +1595,7 @@ func (ar *AgentRunner) runPluginWith(ctx context.Context, snap runSnapshot, name
 	resultsHelper := runner.NewApiHelper(pluginLogger, client, labels, name,
 		runner.WithPolicyPaths(policyPaths),
 		runner.WithSources(sourceOf(plugin.Source, pluginExecutable), policySources),
+		runner.WithEvidenceProps(configRevisionProps(config)...),
 	)
 
 	policyBehaviorProto := policyBehaviorToProto(plugin.PolicyBehavior)
@@ -1635,6 +1637,20 @@ func (ar *AgentRunner) SendHeartbeat(ctx context.Context, staticAgentUUID uuid.U
 	return nil
 }
 
+// configRevisionProps returns the evidence prop naming the applied overlay revision, or nil
+// when the agent runs the file only (R38).
+func configRevisionProps(config *agentConfig) []sdktypes.Property {
+	meta := config.syncInfo()
+	if meta.AppliedRevision <= 0 {
+		return nil
+	}
+	return []sdktypes.Property{{
+		Ns:    runner.PropNamespace,
+		Name:  runner.PropConfigRevision,
+		Value: strconv.FormatInt(meta.AppliedRevision, 10),
+	}}
+}
+
 // buildHeartbeat builds the heartbeat body. When remote configuration is not off it carries
 // the applied revision (0 when running the file only, never null) and the effective digest,
 // which lets the API create the instance row (R11, R45).
@@ -1756,6 +1772,7 @@ func (ar *AgentRunner) buildAgentRunEvidence(now time.Time) (*agentEvidenceCreat
 			End:         now,
 			Expires:     expires,
 			Links:       links,
+			Props:       configRevisionProps(config),
 			Status: sdktypes.ObjectiveStatus{
 				Reason:  reason,
 				Remarks: remarks,

cmd/config.go

@@ -31,7 +31,7 @@ type baseSnapshot struct {
 	warnings []agentconfig.FieldError
 	// skip holds the plugins dropped from the runtime because of a tolerated problem.
 	skip map[string]string
-	// fingerprint identifies the base for the failed backoff.
+	// fingerprint identifies the base for the rejected-revision memory.
 	fingerprint string
 }
 
@@ -240,7 +240,7 @@ func baseFromViper(cmd *cobra.Command, v *viper.Viper, raw []byte) (*baseSnapsho
 		raw:        raw,
 		envSourced: envSourcedPointers(v),
 	}
-	part := partitionByOrigin(declared.Validate())
+	part := partitionByOrigin(declared.Validate(), nil)
 	if len(part.fatal) > 0 {
 		return nil, agentconfig.ValidationErrors(part.fatal)
 	}
@@ -252,15 +252,18 @@ func baseFromViper(cmd *cobra.Command, v *viper.Viper, raw []byte) (*baseSnapsho
 
 // validationPartition is the R34 split of a config's validation errors.
 type validationPartition struct {
+	overlay  []agentconfig.FieldError // touched by the overlay: strict
 	fatal    []agentconfig.FieldError // file-origin, not tolerated: fatal
 	warnings []agentconfig.FieldError // file-origin, tolerated or warn-only: reported
 	skip     map[string]string        // plugin name -> reason, for tolerated (skip) errors
 }
 
-// partitionByOrigin splits the validation errors of the file (R34): tolerated rules become
-// warnings (and the plugin is skipped), warn-only rules become warnings (nothing is skipped or
+// partitionByOrigin splits validation errors by origin (R34). An error at pointer P is
+// overlay-origin when some overlay-touched pointer o equals P, is a prefix of P, or has P as a
+// prefix (segment-wise). Everything else is file-origin: tolerated rules become warnings
+// (and the plugin is skipped), warn-only rules become warnings (nothing is skipped or
 // changed), the rest is fatal.
-func partitionByOrigin(err error) validationPartition {
+func partitionByOrigin(err error, overlayTouched []string) validationPartition {
 	var out validationPartition
 	if err == nil {
 		return out
@@ -272,6 +275,8 @@ func partitionByOrigin(err error) validationPartition {
 	}
 	for _, e := range errs {
 		switch {
+		case touchedByOverlay(e.Path, overlayTouched):
+			out.overlay = append(out.overlay, e)
 		case isToleratedFileRule(e):
 			out.warnings = append(out.warnings, e)
 			if segs := agentconfig.SplitPointer(e.Path); len(segs) >= 2 && segs[0] == "plugins" {
@@ -289,6 +294,19 @@ func partitionByOrigin(err error) validationPartition {
 	return out
 }
 
+// touchedByOverlay compares pointers segment-wise in both directions.
+func touchedByOverlay(ptr string, touched []string) bool {
+	p := agentconfig.SplitPointer(ptr)
+	for _, o := range touched {
+		t := agentconfig.SplitPointer(o)
+		n := min(len(p), len(t))
+		if slices.Equal(p[:n], t[:n]) {
+			return true
+		}
+	}
+	return false
+}
+
 // toRuntime converts a merged, env-resolved declared config into the runtime structs.
 // Disabled plugins and plugins named in skip (R34) are dropped: they get no cron, no download
 // and no run state, but they stay in the declared form and in reports.

cmd/config_test.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"encoding/json"
 	"errors"
 	"os"
 	"path/filepath"
@@ -9,6 +10,7 @@ import (
 
 	"github.com/compliance-framework/api/pkg/agentconfig"
 	"github.com/hashicorp/go-hclog"
+	"google.golang.org/protobuf/proto"
 )
 
 // writeConfigFile writes content to a temp file with the given extension and returns its path.
@@ -65,6 +67,41 @@ func TestLoadBase_WeakDecodingUnchanged(t *testing.T) {
 	}
 }
 
+// TestWeakDecoding_SurvivesUnrelatedOverlay checks that an overlay touching only the schedule
+// leaves the plugin's config and policy_data unchanged on the wire (R51).
+func TestWeakDecoding_SurvivesUnrelatedOverlay(t *testing.T) {
+	base := mustLoadBase(t, "yaml", weakTypedConfig)
+	fileOnly, err := toRuntime(base.declared, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	merged, err := agentconfig.Merge(base.declared, json.RawMessage(`{"plugins":{"aws":{"schedule":"*/5 * * * *"}}}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	withOverlay, err := toRuntime(merged, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(fileOnly.Plugins["aws"].Config, withOverlay.Plugins["aws"].Config) {
+		t.Fatalf("config changed by an unrelated overlay: %#v vs %#v", fileOnly.Plugins["aws"].Config, withOverlay.Plugins["aws"].Config)
+	}
+	a, err := mapToStruct(fileOnly.Plugins["aws"].PolicyData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, err := mapToStruct(withOverlay.Plugins["aws"].PolicyData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !proto.Equal(a, b) {
+		t.Fatalf("policy_data structpb differs: %v vs %v", a, b)
+	}
+	if got := *withOverlay.Plugins["aws"].Schedule; got != "*/5 * * * *" {
+		t.Fatalf("overlay schedule not applied: %q", got)
+	}
+}
+
 func TestEnvSourcedPointers(t *testing.T) {
 	t.Setenv("CCF_PLUGINS_GITHUB_CONFIG_TOKEN", "from-env")
 	base := mustLoadBase(t, "yaml", `
@@ -174,6 +211,16 @@ func TestLoadBase_FileOriginWarnOnly(t *testing.T) {
 		})
 	}
 
+	t.Run("overlay-origin stays strict", func(t *testing.T) {
+		errs := agentconfig.ValidationErrors{
+			{Path: "/verbosity", Code: agentconfig.FieldCodeInvalidValue, Message: "must not be negative"},
+			{Path: "/plugins/ssh/labels/team", Code: agentconfig.FieldCodeEnvLocation, Message: "env"},
+		}
+		p := partitionByOrigin(errs, []string{"/verbosity", "/plugins/ssh/labels/team"})
+		if len(p.overlay) != 2 || len(p.warnings) != 0 {
+			t.Fatalf("overlay-introduced values must be strict, got %#v", p)
+		}
+	})
 }
 
 // TestLoadBase_LoadsAsOnMain: YAML that JSON cannot represent loads, and a key the agent does
@@ -209,6 +256,40 @@ plugins:
 	}
 }
 
+func TestPartitionByOrigin(t *testing.T) {
+	errs := agentconfig.ValidationErrors{
+		{Path: "/plugins/ssh/schedule", Code: agentconfig.FieldCodeCron, Message: "bad cron"},
+		{Path: "/plugins/github/source", Code: agentconfig.FieldCodeRequired, Message: "source required"},
+	}
+	t.Run("overlay touches another field of the same plugin", func(t *testing.T) {
+		p := partitionByOrigin(errs, []string{"/plugins/ssh/labels/team"})
+		if len(p.overlay) != 0 || len(p.warnings) != 1 || len(p.fatal) != 1 {
+			t.Fatalf("unexpected partition %#v", p)
+		}
+		if _, ok := p.skip["ssh"]; !ok {
+			t.Fatalf("expected ssh skipped, got %v", p.skip)
+		}
+	})
+	t.Run("overlay sets the schedule", func(t *testing.T) {
+		p := partitionByOrigin(errs, []string{"/plugins/ssh/schedule"})
+		if len(p.overlay) != 1 || p.overlay[0].Path != "/plugins/ssh/schedule" || len(p.warnings) != 0 {
+			t.Fatalf("unexpected partition %#v", p)
+		}
+	})
+	t.Run("overlay adds the plugin", func(t *testing.T) {
+		p := partitionByOrigin(errs, []string{"/plugins/ssh"})
+		if len(p.overlay) != 1 {
+			t.Fatalf("a prefix pointer must make the error overlay-origin, got %#v", p)
+		}
+	})
+	t.Run("segment-wise, not string-wise", func(t *testing.T) {
+		p := partitionByOrigin(errs, []string{"/plugins/ss"})
+		if len(p.overlay) != 0 {
+			t.Fatalf("/plugins/ss must not match /plugins/ssh, got %#v", p)
+		}
+	})
+}
+
 func TestToRuntime_DisabledPluginDropped(t *testing.T) {
 	base := mustLoadBase(t, "yaml", `
 api: