diff --git a/README.md b/README.md index 5af85be..b7a01bd 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ api: plugins: : # Can have as many of these as you like + enabled: true # Optional: default true; a disabled plugin is not scheduled or downloaded protocol_version: 2 # Optional: Defaults to 1 for backwards compatibility source: labels: diff --git a/cmd/agent.go b/cmd/agent.go index f817c41..4c6aa29 100644 --- a/cmd/agent.go +++ b/cmd/agent.go @@ -13,7 +13,7 @@ import ( "os" "os/exec" "os/signal" - "path" + "path/filepath" "runtime" "sort" "strconv" @@ -78,6 +78,8 @@ type agentEvidenceConfig struct { Interval string `mapstructure:"interval,omitempty"` } +// agentConfig is the RUNTIME form of the configuration, built from the declared form +// (agentconfig.Config) by toRuntime. type agentConfig struct { Daemon bool `mapstructure:"daemon"` Verbosity int32 `mapstructure:"verbosity"` @@ -86,44 +88,13 @@ type agentConfig struct { AgentEvidence *agentEvidenceConfig `mapstructure:"agent_evidence"` } -// logVerbosity reverses our verbosity "increase" to hclog's reversed "decrease." -// 1 for us means INFO. 1 for hclog means trace. -// 3 for us means TRACE. 3 for hclog means INFO. -// You can see hclog's verbosity here: https://github.com/hashicorp/go-hclog/blob/cb8687c9c619227eac510d0a76d23997fb6667d3/logger.go#L25 +// logVerbosity maps our verbosity "increase" onto hclog's levels: our 0/1/2 = Info/Debug/Trace, +// i.e. hclog.Level(Info - v). See hclog's levels here: +// https://github.com/hashicorp/go-hclog/blob/cb8687c9c619227eac510d0a76d23997fb6667d3/logger.go#L25 func (ac *agentConfig) logVerbosity() int32 { return int32(hclog.Info) - ac.Verbosity } -func (ac *agentConfig) validate() error { - if err := ac.ApiConfig.validate(); err != nil { - return err - } - - if _, err := ac.agentEvidenceInterval(); err != nil { - return err - } - - for name, pluginConfig := range ac.Plugins { - if pluginConfig == nil { - return fmt.Errorf("plugin %s has null configuration", name) - } - - if pluginConfig.ProtocolVersion == 0 { - if pluginConfig.protocolSet { - return fmt.Errorf("plugin %s has unsupported protocol_version=%d; supported values are %d and %d", name, pluginConfig.ProtocolVersion, DefaultProtocolVersion, RunnerV2ProtocolVersion) - } - - continue - } - - if !isSupportedProtocolVersion(pluginConfig.ProtocolVersion) { - return fmt.Errorf("plugin %s has unsupported protocol_version=%d; supported values are %d and %d", name, pluginConfig.ProtocolVersion, DefaultProtocolVersion, RunnerV2ProtocolVersion) - } - } - - return nil -} - func (ac *agentConfig) agentEvidenceEnabled() bool { if ac == nil || ac.AgentEvidence == nil || ac.AgentEvidence.Enabled == nil { return true @@ -157,28 +128,6 @@ func (ac *agentConfig) agentEvidenceInterval() (time.Duration, error) { return interval, nil } -func (ac *apiConfig) validate() error { - if ac == nil { - return fmt.Errorf("no api config specified in config") - } - - if strings.TrimSpace(ac.Url) == "" { - return fmt.Errorf("api url must be configured") - } - - if ac.hasPartialAuth() { - return fmt.Errorf("api auth requires both client_id and client_secret when configured") - } - - if ac.hasAuth() { - if _, err := uuid.Parse(strings.TrimSpace(ac.Auth.ClientID)); err != nil { - return fmt.Errorf("api auth client_id must be a valid UUID") - } - } - - return nil -} - func (ac *apiConfig) hasAuth() bool { return ac != nil && ac.Auth != nil && @@ -237,10 +186,7 @@ with plugins to ensure continuous compliance.`, } agentCmd.Flags().CountP("verbose", "v", "Enable verbose output") - viper.BindPFlag("verbose", agentCmd.Flags().Lookup("verbose")) - agentCmd.Flags().BoolP("daemon", "d", false, "Specify to run as a long running daemon") - viper.BindPFlag("daemon", agentCmd.Flags().Lookup("daemon")) agentCmd.Flags().StringP("config", "c", "", "Location of config file") agentCmd.MarkFlagRequired("config") @@ -248,91 +194,6 @@ with plugins to ensure continuous compliance.`, return agentCmd } -func mergeConfig(cmd *cobra.Command, fileConfig *viper.Viper) (*agentConfig, error) { - // For now, we are reading from a file. This will probably be updated to a remote source soon. - - // Daemon has a default false value, which will override all values passed through Viper. - // We need to check whether it was actually passed `Changed()`, and then merge its value into our config. - if cmd.Flags().Changed("daemon") { - isDaemon, err := cmd.Flags().GetBool("daemon") - if err != nil { - return nil, err - } - - err = fileConfig.MergeConfigMap(map[string]interface{}{ - "daemon": isDaemon, - }) - if err != nil { - return nil, err - } - } - - if cmd.Flags().Changed("verbose") { - verbosity, err := cmd.Flags().GetCount("verbose") - if err != nil { - return nil, err - } - err = fileConfig.MergeConfigMap(map[string]interface{}{ - "verbosity": verbosity, - }) - if err != nil { - return nil, err - } - } - - config := &agentConfig{} - err := fileConfig.Unmarshal(config) - - if err != nil { - return nil, err - } - - markExplicitPluginProtocols(fileConfig, config) - updateAllPluginProtocols(config) - - return config, nil -} - -func bindAgentEnv(config *viper.Viper) error { - for key, envVar := range map[string]string{ - "api.auth.client_id": "CCF_API_AUTH_CLIENT_ID", - "api.auth.client_secret": "CCF_API_AUTH_CLIENT_SECRET", - } { - if err := config.BindEnv(key, envVar); err != nil { - return err - } - } - - return nil -} - -func markExplicitPluginProtocols(fileConfig *viper.Viper, config *agentConfig) { - rawPlugins := fileConfig.GetStringMap("plugins") - for name, rawPlugin := range rawPlugins { - pluginConfig, ok := config.Plugins[name] - if rawPlugin == nil { - if config.Plugins == nil { - config.Plugins = map[string]*agentPlugin{} - } - if !ok { - config.Plugins[name] = nil - } - continue - } - - if !ok || pluginConfig == nil { - continue - } - - pluginMap, ok := rawPlugin.(map[string]interface{}) - if !ok { - continue - } - - _, pluginConfig.protocolSet = pluginMap["protocol_version"] - } -} - func updateAllPluginProtocols(agentConfig *agentConfig) { for _, pluginConfig := range agentConfig.Plugins { if pluginConfig != nil && !pluginConfig.protocolSet && pluginConfig.ProtocolVersion == 0 { @@ -412,22 +273,21 @@ func configureRunner(name string, runnerInstance runner.RunnerV2, config agentPl return err } -func loadConfig(cmd *cobra.Command, v *viper.Viper) (*agentConfig, error) { - err := v.ReadInConfig() - if err != nil { - return nil, err - } - - config, err := mergeConfig(cmd, v) +// loadConfig reads and validates the config file and builds its runtime form. Tolerated file +// problems (R34) are logged; the plugins they affect are skipped. +func loadConfig(cmd *cobra.Command, configPath string, logger hclog.Logger) (*agentConfig, error) { + base, err := loadBase(cmd, configPath) if err != nil { return nil, err } - - err = config.validate() - if err != nil { - return nil, err + for _, w := range base.warnings { + if isToleratedFileRule(w) { + logger.Warn("Ignoring a problem in the config file; the plugin is skipped", "path", w.Path, "error", w.Message) + continue + } + logger.Warn("Ignoring a problem in the config file; the value is kept unchanged", "path", w.Path, "error", w.Message) } - return config, nil + return toRuntime(base.declared, base.skip) } // Main the entrypoint for the `agent` command @@ -435,22 +295,8 @@ func loadConfig(cmd *cobra.Command, v *viper.Viper) (*agentConfig, error) { // It will read the configuration file, and then run the agent. Various command line flags can // be used to override the config file. func agentRunner(cmd *cobra.Command, args []string) error { - configPath := cmd.Flag("config").Value.String() - - if !path.IsAbs(configPath) { - workDir, err := os.Getwd() - if err != nil { - return err - } - configPath = path.Join(workDir, configPath) - } - - v := viper.New() - v.SetConfigFile(configPath) - v.SetEnvPrefix("CCF") - v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) - v.AutomaticEnv() - if err := bindAgentEnv(v); err != nil { + configPath, err := filepath.Abs(cmd.Flag("config").Value.String()) + if err != nil { return err } @@ -465,18 +311,21 @@ func agentRunner(cmd *cobra.Command, args []string) error { ctx, configCancel := context.WithCancel(context.Background()) defer configCancel() - v.OnConfigChange(func(in fsnotify.Event) { + // The watcher only signals; every load builds a fresh viper (loadBase). + w := viper.New() + w.SetConfigFile(configPath) + w.OnConfigChange(func(in fsnotify.Event) { // We want to wait for any running agent processes to finish first. logger.Debug("config file changed", "path", in.Name) configCancel() }) - v.WatchConfig() + w.WatchConfig() // For the daemon, we run the agent continuously. // It will exit as soon as the config changes, and then start again with new configs set. for { ctx, configCancel = context.WithCancel(context.Background()) - config, err := loadConfig(cmd, v) + config, err := loadConfig(cmd, configPath, logger) if err != nil { logger.Error("Error loading new config", "error", err) panic(err) diff --git a/cmd/agent_test.go b/cmd/agent_test.go index 5d72fd6..45294a0 100644 --- a/cmd/agent_test.go +++ b/cmd/agent_test.go @@ -17,10 +17,12 @@ import ( "github.com/compliance-framework/agent/runner" "github.com/compliance-framework/agent/runner/proto" + "github.com/compliance-framework/api/pkg/agentconfig" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/uuid" "github.com/hashicorp/go-hclog" hplugin "github.com/hashicorp/go-plugin" + "github.com/spf13/cobra" "github.com/spf13/viper" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -186,15 +188,8 @@ plugins: t.Fatalf("Error reading config: %v", err) } - config := &agentConfig{} - err = v.Unmarshal(config) - if err != nil { - t.Fatalf("Error unmarshalling config: %v", err) - } - markExplicitPluginProtocols(v, config) - updateAllPluginProtocols(config) - - if err = config.validate(); (err == nil) != test.valid { + _, err = baseFromViper(AgentCmd(), v, []byte(test.configYamlContent)) + if (err == nil) != test.valid { t.Errorf("Expected validity of config to be %v, got %v", test.valid, err) } }) @@ -340,17 +335,12 @@ plugins: t.Fatalf("Error reading config: %v", err) } - config, err := mergeConfig(AgentCmd(), v) - if err != nil { - t.Fatalf("Error merging config: %v", err) - } - - err = config.validate() + _, err = baseFromViper(AgentCmd(), v, nil) if err == nil { t.Fatal("expected validate to fail when only one api auth env var is set") } - if err.Error() != "api auth requires both client_id and client_secret when configured" { - t.Fatalf("expected validate error %q, got %q", "api auth requires both client_id and client_secret when configured", err.Error()) + if err.Error() != "/api/auth: api auth requires both client_id and client_secret when configured" { + t.Fatalf("expected validate error %q, got %q", "/api/auth: api auth requires both client_id and client_secret when configured", err.Error()) } }) } @@ -452,17 +442,12 @@ func TestMergeConfig_RejectsUnsupportedExplicitProtocolVersion(t *testing.T) { t.Fatalf("Error reading config: %v", err) } - config, err := mergeConfig(AgentCmd(), v) - if err != nil { - t.Fatalf("Error merging config: %v", err) - } - - err = config.validate() + _, err = baseFromViper(AgentCmd(), v, nil) if err == nil { t.Fatalf("Expected config validation to fail for unsupported protocol version") } - expected := "plugin plugin-with-invalid-version has unsupported protocol_version=100; supported values are 1 and 2" + expected := "/plugins/plugin-with-invalid-version/protocol_version: must be 1 or 2 (0 or unset = auto)" if err.Error() != expected { t.Fatalf("Expected error %q, got %q", expected, err.Error()) } @@ -476,12 +461,7 @@ func TestMergeConfig_RejectsExplicitZeroProtocolVersion(t *testing.T) { t.Fatalf("Error reading config: %v", err) } - config, err := mergeConfig(AgentCmd(), v) - if err != nil { - t.Fatalf("Error merging config: %v", err) - } - - err = config.validate() + _, err = baseFromViper(AgentCmd(), v, nil) if err == nil { t.Fatalf("Expected config validation to fail for explicit zero protocol version") } @@ -500,17 +480,12 @@ func TestMergeConfig_RejectsNullPluginConfiguration(t *testing.T) { t.Fatalf("Error reading config: %v", err) } - config, err := mergeConfig(AgentCmd(), v) - if err != nil { - t.Fatalf("Error merging config: %v", err) - } - - err = config.validate() + _, err = baseFromViper(AgentCmd(), v, nil) if err == nil { t.Fatalf("Expected config validation to fail for null plugin configuration") } - expected := "plugin null-plugin has null configuration" + expected := "/plugins/null-plugin: plugin \"null-plugin\" has no configuration" if err.Error() != expected { t.Fatalf("Expected error %q, got %q", expected, err.Error()) } @@ -2056,7 +2031,7 @@ func TestAgentEvidenceConfigDefaultsAndValidation(t *testing.T) { config := &agentConfig{ ApiConfig: &apiConfig{Url: "http://localhost:8080"}, } - if err := config.validate(); err != nil { + if err := validateRuntimeForTest(config); err != nil { t.Fatalf("expected no-plugin config to be valid: %v", err) } if !config.agentEvidenceEnabled() { @@ -2074,11 +2049,36 @@ func TestAgentEvidenceConfigDefaultsAndValidation(t *testing.T) { } config.AgentEvidence = &agentEvidenceConfig{Interval: "not-a-duration"} - if err := config.validate(); err == nil { + if err := validateRuntimeForTest(config); err == nil { t.Fatalf("expected invalid interval to fail validation") } } +// mergeConfig decodes the declared config from v and converts it to the runtime form without +// validating it (the old mergeConfig contract, kept for these tests). +func mergeConfig(cmd *cobra.Command, v *viper.Viper) (*agentConfig, error) { + declared, err := declaredFromViper(cmd, v) + if err != nil { + return nil, err + } + return toRuntime(declared, nil) +} + +// validateRuntimeForTest validates the declared equivalent of a runtime config. +func validateRuntimeForTest(config *agentConfig) error { + declared := agentconfig.Config{Daemon: config.Daemon, Verbosity: config.Verbosity} + if config.ApiConfig != nil { + declared.API = &agentconfig.APIConfig{URL: config.ApiConfig.Url} + if config.ApiConfig.Auth != nil { + declared.API.Auth = &agentconfig.APIAuth{ClientID: config.ApiConfig.Auth.ClientID, ClientSecret: config.ApiConfig.Auth.ClientSecret} + } + } + if config.AgentEvidence != nil { + declared.AgentEvidence = &agentconfig.EvidenceConfig{Enabled: config.AgentEvidence.Enabled, EmitOnRunCompletion: config.AgentEvidence.EmitOnRunCompletion, Interval: config.AgentEvidence.Interval} + } + return declared.Validate() +} + func newTestAgentConfig(baseURL string, auth *apiAuthConfig) *agentConfig { return &agentConfig{ ApiConfig: &apiConfig{ diff --git a/cmd/config.go b/cmd/config.go new file mode 100644 index 0000000..232e10e --- /dev/null +++ b/cmd/config.go @@ -0,0 +1,346 @@ +package cmd + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + + "github.com/compliance-framework/api/pkg/agentconfig" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// baseSnapshot is one load of the local configuration: the file merged with CLI flags and +// bound environment variables. It is immutable once built. +// +// declared is the declared form (agentconfig.Config): it is what gets merged, classified, +// validated, redacted, digested and reported. The runtime form (*agentConfig) is built from +// it by toRuntime. +type baseSnapshot struct { + declared agentconfig.Config // file ⊕ CLI flags ⊕ bound env + raw []byte // exact bytes read (one read per load) + // warnings are tolerated file-origin problems (R34): reported, never fatal. + warnings []agentconfig.FieldError + // skip holds the plugins dropped from the runtime because of a tolerated problem. + skip map[string]string +} + +// toleratedFileRules are the validation rules whose failure is non-fatal when the value comes +// from the local file (R34). Today a bad file schedule only logs "Error adding plugin +// schedule" and the plugin never runs; everything else that fails validation fails startup. +// This list is closed: rules for new features never go here, because no existing file can +// depend on them. +var toleratedFileRules = []*regexp.Regexp{ + regexp.MustCompile(`^/plugins/[^/]+/schedule$`), +} + +func isToleratedFileRule(e agentconfig.FieldError) bool { + for _, re := range toleratedFileRules { + if re.MatchString(e.Path) { + return true + } + } + return false +} + +// isWarnOnlyFileRule reports whether a file-origin error is a warning that neither skips a +// plugin nor changes the value (R34, R51; owner review of agent#95). These are values that +// load on main with a meaning the agent keeps: +// - a negative verbosity: hclog.Info - v, i.e. a quieter agent (-1 = Warn); +// - a literal ${env:...} outside plugins.*.config (labels, policy_data, ...): an opaque +// string handed to the plugin or to Rego, never resolved. +func isWarnOnlyFileRule(e agentconfig.FieldError) bool { + return e.Path == "/verbosity" || e.Code == agentconfig.FieldCodeEnvLocation +} + +// newAgentViper builds a fresh viper for one load (R32): the watcher goroutine and the loader +// never share an instance. +func newAgentViper(configPath string) (*viper.Viper, error) { + ext := configExt(configPath) + v := viper.New() + v.SetConfigType(ext) + v.SetEnvPrefix("CCF") + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + v.AutomaticEnv() + if err := bindAgentEnv(v); err != nil { + return nil, err + } + return v, nil +} + +func configExt(configPath string) string { + return strings.ToLower(strings.TrimPrefix(filepath.Ext(configPath), ".")) +} + +func bindAgentEnv(config *viper.Viper) error { + for key, envVar := range map[string]string{ + "api.auth.client_id": "CCF_API_AUTH_CLIENT_ID", + "api.auth.client_secret": "CCF_API_AUTH_CLIENT_SECRET", + } { + if err := config.BindEnv(key, envVar); err != nil { + return err + } + } + + return nil +} + +// applyFlagOverrides merges CLI flags that were explicitly set into the viper config. +func applyFlagOverrides(cmd *cobra.Command, v *viper.Viper) error { + // Daemon has a default false value, which will override all values passed through Viper. + // We need to check whether it was actually passed `Changed()`, and then merge its value into our config. + if flag := cmd.Flags().Lookup("daemon"); flag != nil && flag.Changed { + isDaemon, err := cmd.Flags().GetBool("daemon") + if err != nil { + return err + } + if err := v.MergeConfigMap(map[string]interface{}{"daemon": isDaemon}); err != nil { + return err + } + } + + if flag := cmd.Flags().Lookup("verbose"); flag != nil && flag.Changed { + verbosity, err := cmd.Flags().GetCount("verbose") + if err != nil { + return err + } + if err := v.MergeConfigMap(map[string]interface{}{"verbosity": verbosity}); err != nil { + return err + } + } + return nil +} + +// declaredFromViper decodes the declared config from a viper that has read the file. It uses +// viper's default (weakly typed) decoder, exactly as the agent always has (R51): for example +// a YAML `false` plugin config value becomes "0" and a number becomes its decimal string. +func declaredFromViper(cmd *cobra.Command, v *viper.Viper) (agentconfig.Config, error) { + if err := applyFlagOverrides(cmd, v); err != nil { + return agentconfig.Config{}, err + } + + var declared agentconfig.Config + if err := v.Unmarshal(&declared); err != nil { + return agentconfig.Config{}, err + } + + markNullPlugins(v, &declared) + if err := checkExplicitZeroProtocol(v, declared); err != nil { + return agentconfig.Config{}, err + } + return declared, nil +} + +// markNullPlugins keeps `plugins: {x: null}` as a nil entry so validation rejects it, as it +// always has. +func markNullPlugins(v *viper.Viper, declared *agentconfig.Config) { + for name, rawPlugin := range v.GetStringMap("plugins") { + if rawPlugin != nil { + continue + } + if declared.Plugins == nil { + declared.Plugins = map[string]*agentconfig.Plugin{} + } + if _, ok := declared.Plugins[name]; !ok { + declared.Plugins[name] = nil + } + } +} + +// checkExplicitZeroProtocol rejects an explicit `protocol_version: 0` in the file (R9). In the +// declared form 0 means "auto", so the explicit value is only visible to viper. +func checkExplicitZeroProtocol(v *viper.Viper, declared agentconfig.Config) error { + names := make([]string, 0) + for name, rawPlugin := range v.GetStringMap("plugins") { + pluginMap, ok := rawPlugin.(map[string]interface{}) + if !ok { + continue + } + if _, set := pluginMap["protocol_version"]; !set { + continue + } + if p := declared.Plugins[name]; p != nil && p.ProtocolVersion == 0 { + names = append(names, name) + } + } + if len(names) == 0 { + return nil + } + slices.Sort(names) + return fmt.Errorf("plugin %s has unsupported protocol_version=0; supported values are %d and %d", names[0], DefaultProtocolVersion, RunnerV2ProtocolVersion) +} + +// loadBase reads and validates the local configuration. It builds a fresh viper per call +// (R32). A returned error means the file is unusable (fatal at startup; keep last-known-good +// on reload). Tolerated file problems (R34) are returned as warnings and skipped plugins. +func loadBase(cmd *cobra.Command, configPath string) (*baseSnapshot, error) { + raw, err := os.ReadFile(configPath) + if err != nil { + return nil, err + } + v, err := newAgentViper(configPath) + if err != nil { + return nil, err + } + if err := v.ReadConfig(bytes.NewReader(raw)); err != nil { + return nil, err + } + return baseFromViper(cmd, v, raw) +} + +// baseFromViper finishes loadBase on a viper that has read raw. +func baseFromViper(cmd *cobra.Command, v *viper.Viper, raw []byte) (*baseSnapshot, error) { + declared, err := declaredFromViper(cmd, v) + if err != nil { + return nil, err + } + + base := &baseSnapshot{ + declared: declared, + raw: raw, + } + part := partitionByOrigin(declared.Validate()) + if len(part.fatal) > 0 { + return nil, agentconfig.ValidationErrors(part.fatal) + } + base.warnings = part.warnings + base.skip = part.skip + return base, nil +} + +// validationPartition is the R34 split of a config's validation errors. +type validationPartition struct { + fatal []agentconfig.FieldError // file-origin, not tolerated: fatal + warnings []agentconfig.FieldError // file-origin, tolerated or warn-only: reported + skip map[string]string // plugin name -> reason, for tolerated (skip) errors +} + +// partitionByOrigin splits the validation errors of the file (R34): tolerated rules become +// warnings (and the plugin is skipped), warn-only rules become warnings (nothing is skipped or +// changed), the rest is fatal. +func partitionByOrigin(err error) validationPartition { + var out validationPartition + if err == nil { + return out + } + var errs agentconfig.ValidationErrors + if !errors.As(err, &errs) { + out.fatal = []agentconfig.FieldError{{Path: "", Code: agentconfig.FieldCodeInvalidValue, Message: err.Error()}} + return out + } + for _, e := range errs { + switch { + case isToleratedFileRule(e): + out.warnings = append(out.warnings, e) + if segs := agentconfig.SplitPointer(e.Path); len(segs) >= 2 && segs[0] == "plugins" { + if out.skip == nil { + out.skip = map[string]string{} + } + out.skip[segs[1]] = e.Message + } + case isWarnOnlyFileRule(e): + out.warnings = append(out.warnings, e) + default: + out.fatal = append(out.fatal, e) + } + } + return out +} + +// toRuntime converts a merged, env-resolved declared config into the runtime structs. +// Disabled plugins and plugins named in skip (R34) are dropped: they get no cron, no download +// and no run state, but they stay in the declared form and in reports. +func toRuntime(c agentconfig.Config, skip map[string]string) (*agentConfig, error) { + out := &agentConfig{ + Daemon: c.Daemon, + Verbosity: c.Verbosity, + Plugins: map[string]*agentPlugin{}, + } + if c.API != nil { + out.ApiConfig = &apiConfig{Url: c.API.URL} + if c.API.Auth != nil { + out.ApiConfig.Auth = &apiAuthConfig{ + ClientID: c.API.Auth.ClientID, + ClientSecret: c.API.Auth.ClientSecret, + } + } + } + if c.AgentEvidence != nil { + out.AgentEvidence = &agentEvidenceConfig{ + Enabled: cloneBool(c.AgentEvidence.Enabled), + EmitOnRunCompletion: cloneBool(c.AgentEvidence.EmitOnRunCompletion), + Interval: c.AgentEvidence.Interval, + } + } + for name, p := range c.Plugins { + if p == nil { + return nil, fmt.Errorf("plugin %s has null configuration", name) + } + if !p.IsEnabled() { + continue + } + if _, skipped := skip[name]; skipped { + continue + } + rp := &agentPlugin{ + ProtocolVersion: p.ProtocolVersion, + protocolSet: p.ProtocolVersion != 0, + Source: p.Source, + Config: agentPluginConfig(copyStringMapKeepEmpty(p.Config)), + Labels: copyStringMapKeepEmpty(p.Labels), + PolicyData: p.PolicyData, + PolicyBehavior: p.PolicyBehavior, + } + if p.Schedule != nil { + s := *p.Schedule + rp.Schedule = &s + } + if p.Policies != nil { + rp.Policies = make([]agentPolicy, 0, len(p.Policies)) + for _, e := range p.Policies { + rp.Policies = append(rp.Policies, agentPolicy(e)) + } + } + out.Plugins[name] = rp + } + updateAllPluginProtocols(out) + return out, nil +} + +func cloneBool(b *bool) *bool { + if b == nil { + return nil + } + v := *b + return &v +} + +// copyStringMapKeepEmpty copies m, keeping nil as nil and empty as empty. +func copyStringMapKeepEmpty(m map[string]string) map[string]string { + if m == nil { + return nil + } + out := make(map[string]string, len(m)) + for k, v := range m { + out[k] = v + } + return out +} + +// validateAPIConfig validates a runtime api block with the shared rules (api.url required, +// both or neither credential, client_id a UUID). +func validateAPIConfig(config *apiConfig) error { + declared := agentconfig.Config{} + if config != nil { + declared.API = &agentconfig.APIConfig{URL: config.Url} + if config.Auth != nil { + declared.API.Auth = &agentconfig.APIAuth{ClientID: config.Auth.ClientID, ClientSecret: config.Auth.ClientSecret} + } + } + return declared.Validate() +} diff --git a/cmd/config_golden_test.go b/cmd/config_golden_test.go index d738093..594d127 100644 --- a/cmd/config_golden_test.go +++ b/cmd/config_golden_test.go @@ -3,10 +3,7 @@ package cmd import ( "os" "path/filepath" - "strings" "testing" - - "github.com/spf13/viper" ) // goldenHashFixtures are config files whose agentConfigurationHash was recorded BEFORE the @@ -121,11 +118,14 @@ func loadGoldenFixture(t *testing.T, yaml string) *agentConfig { if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { t.Fatalf("write fixture: %v", err) } - v := newGoldenViper(t, path) - config, err := loadConfig(AgentCmd(), v) + base, err := loadBase(AgentCmd(), path) if err != nil { t.Fatalf("load fixture: %v", err) } + config, err := toRuntime(base.declared, base.skip) + if err != nil { + t.Fatalf("runtime fixture: %v", err) + } return config } @@ -142,16 +142,3 @@ func TestAgentConfigurationHashGolden(t *testing.T) { }) } } - -func newGoldenViper(t *testing.T, path string) *viper.Viper { - t.Helper() - v := viper.New() - v.SetConfigFile(path) - v.SetEnvPrefix("CCF") - v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) - v.AutomaticEnv() - if err := bindAgentEnv(v); err != nil { - t.Fatalf("bind env: %v", err) - } - return v -} diff --git a/cmd/config_test.go b/cmd/config_test.go new file mode 100644 index 0000000..38ba437 --- /dev/null +++ b/cmd/config_test.go @@ -0,0 +1,236 @@ +package cmd + +import ( + "errors" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/compliance-framework/api/pkg/agentconfig" + "github.com/hashicorp/go-hclog" +) + +// writeConfigFile writes content to a temp file with the given extension and returns its path. +func writeConfigFile(t *testing.T, ext, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "config."+ext) + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + return path +} + +func mustLoadBase(t *testing.T, ext, content string) *baseSnapshot { + t.Helper() + base, err := loadBase(AgentCmd(), writeConfigFile(t, ext, content)) + if err != nil { + t.Fatalf("loadBase: %v", err) + } + return base +} + +const weakTypedConfig = ` +api: + url: http://localhost:8080 +plugins: + aws: + source: ./plugin-aws + schedule: "0 * * * *" + config: + collect_ip_allow_list: false + account_id: 123456789012 + port: 22 + policy_data: + max_auth_tries: 3 + ratio: 0.5 + nested: + list: [1, 2] +` + +// TestLoadBase_WeakDecodingUnchanged checks R51: the file keeps viper's weak decoding exactly +// as today, with no warning and no skip. +func TestLoadBase_WeakDecodingUnchanged(t *testing.T) { + base := mustLoadBase(t, "yaml", weakTypedConfig) + if len(base.warnings) != 0 || len(base.skip) != 0 { + t.Fatalf("expected no warnings or skips, got %v %v", base.warnings, base.skip) + } + rt, err := toRuntime(base.declared, base.skip) + if err != nil { + t.Fatal(err) + } + want := agentPluginConfig{"collect_ip_allow_list": "0", "account_id": "123456789012", "port": "22"} + if got := rt.Plugins["aws"].Config; !reflect.DeepEqual(got, want) { + t.Fatalf("plugin config changed: got %#v want %#v", got, want) + } +} + +func TestLoadBase_BadFileScheduleIsTolerated(t *testing.T) { + base := mustLoadBase(t, "yaml", ` +api: + url: http://localhost:8080 +plugins: + ssh: + source: ./plugin-ssh + schedule: "not a cron" + github: + source: ./plugin-github +`) + if len(base.warnings) != 1 || base.warnings[0].Path != "/plugins/ssh/schedule" || base.warnings[0].Code != agentconfig.FieldCodeCron { + t.Fatalf("expected one cron warning, got %#v", base.warnings) + } + rt, err := toRuntime(base.declared, base.skip) + if err != nil { + t.Fatal(err) + } + if _, ok := rt.Plugins["ssh"]; ok { + t.Fatalf("expected ssh to be skipped") + } + if _, ok := rt.Plugins["github"]; !ok { + t.Fatalf("expected github to run") + } + if _, ok := base.declared.Plugins["ssh"]; !ok { + t.Fatalf("skipped plugin must stay in the declared (reported) config") + } +} + +// TestLoadBase_FileOriginWarnOnly pins R34/R51: values that load on main keep loading. They +// are reported as warnings, the value is unchanged and no plugin is skipped. +func TestLoadBase_FileOriginWarnOnly(t *testing.T) { + tests := []struct { + name string + content string + wantPath string + check func(t *testing.T, rt *agentConfig) + }{ + { + name: "negative verbosity", + content: "verbosity: -1\napi:\n url: http://localhost:8080\nplugins:\n ssh:\n source: ./plugin-ssh\n", + wantPath: "/verbosity", + check: func(t *testing.T, rt *agentConfig) { + if rt.Verbosity != -1 || rt.logVerbosity() != int32(hclog.Warn) { + t.Fatalf("verbosity -1 must stay Warn level, got %d", rt.Verbosity) + } + }, + }, + { + name: "literal env placeholder in labels", + content: "api:\n url: http://localhost:8080\nplugins:\n ssh:\n source: ./plugin-ssh\n labels:\n team: \"${env:TEAM}\"\n", + wantPath: "/plugins/ssh/labels/team", + check: func(t *testing.T, rt *agentConfig) { + if got := rt.Plugins["ssh"].Labels["team"]; got != "${env:TEAM}" { + t.Fatalf("the label must be passed through unchanged, got %q", got) + } + }, + }, + { + name: "literal env placeholder in policy_data", + content: "api:\n url: http://localhost:8080\nplugins:\n ssh:\n source: ./plugin-ssh\n policy_data:\n url: \"${env:URL}\"\n", + wantPath: "/plugins/ssh/policy_data/url", + check: func(t *testing.T, rt *agentConfig) { + if got := rt.Plugins["ssh"].PolicyData["url"]; got != "${env:URL}" { + t.Fatalf("policy_data must be passed through unchanged, got %v", got) + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + base := mustLoadBase(t, "yaml", tt.content) + if len(base.warnings) != 1 || base.warnings[0].Path != tt.wantPath { + t.Fatalf("expected one warning at %s, got %#v", tt.wantPath, base.warnings) + } + if len(base.skip) != 0 { + t.Fatalf("a warn-only problem must not skip a plugin, got %v", base.skip) + } + rt, err := toRuntime(base.declared, base.skip) + if err != nil { + t.Fatal(err) + } + if _, ok := rt.Plugins["ssh"]; !ok { + t.Fatal("the plugin must run") + } + tt.check(t, rt) + }) + } + +} + +// TestLoadBase_LoadsAsOnMain: YAML that JSON cannot represent loads, and a key the agent does +// not know (here a leftover policy_bundles block) is ignored, as on main. +func TestLoadBase_LoadsAsOnMain(t *testing.T) { + base := mustLoadBase(t, "yaml", "api:\n url: http://localhost:8080\nplugins:\n ssh:\n source: ./plugin-ssh\n policy_data:\n ratio: .nan\n max: .inf\n") + if base.declared.Plugins["ssh"] == nil { + t.Fatalf("plugin ssh missing: %#v", base.declared.Plugins) + } + base = mustLoadBase(t, "yaml", "api:\n url: http://localhost:8080\npolicy_bundles:\n ssh:\n modules:\n a.rego: package a\nplugins:\n ssh:\n source: ./plugin-ssh\n") + if base.declared.Plugins["ssh"] == nil || len(base.warnings) != 0 { + t.Fatalf("an unknown key must be ignored: %#v %#v", base.declared.Plugins, base.warnings) + } +} + +func TestLoadBase_MissingAPIURLIsFatal(t *testing.T) { + _, err := loadBase(AgentCmd(), writeConfigFile(t, "yaml", ` +api: + auth: + client_id: 123e4567-e89b-12d3-a456-426614174000 + client_secret: s +plugins: + ssh: + source: ./plugin-ssh + schedule: "not a cron" +`)) + var verrs agentconfig.ValidationErrors + if !errors.As(err, &verrs) { + t.Fatalf("expected validation errors, got %v", err) + } + if len(verrs) != 1 || verrs[0].Path != "/api/url" { + t.Fatalf("expected only the api.url error to be fatal, got %v", verrs) + } +} + +func TestToRuntime_DisabledPluginDropped(t *testing.T) { + base := mustLoadBase(t, "yaml", ` +api: + url: http://localhost:8080 +plugins: + ssh: + source: ./plugin-ssh + enabled: false + github: + source: ./plugin-github +`) + rt, err := toRuntime(base.declared, nil) + if err != nil { + t.Fatal(err) + } + if _, ok := rt.Plugins["ssh"]; ok { + t.Fatalf("disabled plugin must not be in the runtime config") + } + if p := base.declared.Plugins["ssh"]; p == nil || p.IsEnabled() { + t.Fatalf("disabled plugin must stay declared, got %#v", p) + } +} + +func TestToRuntime_ProtocolVersion(t *testing.T) { + base := mustLoadBase(t, "yaml", ` +api: + url: http://localhost:8080 +plugins: + auto: + source: ./plugin-a + pinned: + source: ./plugin-b + protocol_version: 2 +`) + rt, err := toRuntime(base.declared, nil) + if err != nil { + t.Fatal(err) + } + if p := rt.Plugins["auto"]; p.protocolSet || p.ProtocolVersion != DefaultProtocolVersion { + t.Fatalf("auto plugin: %#v", p) + } + if p := rt.Plugins["pinned"]; !p.protocolSet || p.ProtocolVersion != RunnerV2ProtocolVersion { + t.Fatalf("pinned plugin: %#v", p) + } +} diff --git a/cmd/submit_evidence.go b/cmd/submit_evidence.go index 853ceab..16c0b5f 100644 --- a/cmd/submit_evidence.go +++ b/cmd/submit_evidence.go @@ -363,7 +363,7 @@ func submitEvidenceAPIConfig(apiURLFlag string) (*apiConfig, error) { if config.Auth.ClientID == "" && config.Auth.ClientSecret == "" { config.Auth = nil } - if err := config.validate(); err != nil { + if err := validateAPIConfig(config); err != nil { return nil, err } return config, nil diff --git a/docs/configuration.md b/docs/configuration.md index e762ef9..30b4fb7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -158,3 +158,28 @@ The `log_level` is one of the following, defaulting to `0` if not specified: - 0: Shows all ERROR, WARN and INFO - 1: Shows all of 0 plus DEBUG logs - 2: Shows all of 1 plus TRACE logs + +## Plugin `enabled` + +```yaml +plugins: + : + enabled: false # default true +``` + +A disabled plugin gets no schedule, no download and no run state, but it stays in the configuration the agent reports. + +## Typing of plugin values + +Values in the config file keep viper's weak typing exactly as before: `collect_ip_allow_list: false` reaches the plugin +as `"0"`, `account_id: 123456789012` as `"123456789012"`, and `port: 22` as `"22"`. + +Viper lowercases keys and splits them on dots. Plugin names and config keys in the file are therefore lowercase and +cannot contain dots. + +## Tolerated file problems + +A plugin `schedule` in the file that does not parse does not stop the agent: that plugin is skipped, the others run, +and the problem is logged as a warning (R34). A few other file values that always loaded are also only +warnings, and are kept unchanged: a negative `verbosity` (`-1` logs WARN and above) and a literal `${env:...}` outside +`plugins.*.config`. Every other invalid value in the file (for example a missing `api.url`) still fails startup. diff --git a/go.mod b/go.mod index 0461ee6..afe0637 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/compliance-framework/agent go 1.26.1 require ( - github.com/compliance-framework/api v0.20.0 + github.com/compliance-framework/api v0.21.0 github.com/compliance-framework/gooci v0.0.6 github.com/coreos/go-systemd/v22 v22.7.0 github.com/defenseunicorns/go-oscal v0.7.0 @@ -59,7 +59,7 @@ require ( github.com/google/go-cmp v0.7.0 // indirect github.com/hashicorp/yamux v0.1.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/klauspost/compress v1.18.4 // indirect + github.com/klauspost/compress v1.18.7 // indirect github.com/lestrrat-go/blackmagic v1.0.4 // indirect github.com/lestrrat-go/dsig v1.0.0 // indirect github.com/lestrrat-go/dsig-secp256k1 v1.0.0 // indirect diff --git a/go.sum b/go.sum index db23ece..3695fe3 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/compliance-framework/api v0.20.0 h1:/61+MLsNOkn2sX/kOzO2z8Q3dfk2dAmSzkx+RBLM118= -github.com/compliance-framework/api v0.20.0/go.mod h1:R0MBpd7m1rFeZIYB6kAm0u8o7vEjzMUGYIVeDRxr6Ao= +github.com/compliance-framework/api v0.21.0 h1:FFkeg9tUrIPBOt0pbH10NJhoOcdYYhdepyxDltA6mg4= +github.com/compliance-framework/api v0.21.0/go.mod h1:TrmFnqr+UBvgvKWspYVEnmZSLMyoPbc2i2/jPK7EUww= github.com/compliance-framework/gooci v0.0.6 h1:61N3igJyGdSO5hYD4ODgQ6YhzixbK6mMCzsNLyigBQ4= github.com/compliance-framework/gooci v0.0.6/go.mod h1:vbiRPS2mbxW2VIKhpkOOK6uftKjv9l3fYOr3m+ufwZA= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -220,8 +220,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= -github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= +github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -268,12 +268,12 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= -github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= -github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= -github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= +github.com/moby/sys/user v0.4.1 h1:RgjRlaDKi/Xmyrz4t8lyzXT6v2ooFeO/7xtchmhVWE0= +github.com/moby/sys/user v0.4.1/go.mod h1:E9QsW5WRe1kUAf7kW8hXKwu1uhsZEAdPLYHYSDudF4Y= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= diff --git a/internal/oci.go b/internal/oci.go index 704dd5a..3f24f79 100644 --- a/internal/oci.go +++ b/internal/oci.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" + "github.com/compliance-framework/api/pkg/agentconfig" "github.com/compliance-framework/gooci/pkg/oci" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" @@ -18,10 +19,11 @@ import ( "github.com/hashicorp/go-hclog" ) +// IsOCI reports whether source parses as an OCI tag with strict validation, which is what our +// downloader supports. It delegates to the shared agentconfig rule so the agent and the API +// classify sources identically (R3). func IsOCI(source string) bool { - // Check whether this can be parsed as an OCI tag, which is what our downloader supports. - _, err := name.NewTag(source, name.StrictValidation) - return err == nil + return agentconfig.IsOCISource(source) } func GetAnnotations(ctx context.Context, source string, option ...remote.Option) (map[string]string, error) {