diff --git a/CHANGELOG.md b/CHANGELOG.md index 16d2970..9973773 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,26 @@ limit is used; `-t json|yaml` prints the platform's summary. `license report` downloads the license usage report, a zip archive, under the platform's name without overwriting a file, or to `-o`. Both need an admin access token. +- `export --tenant -d dir` writes the tenant's configuration, to keep in Git: experiment + templates, environments, teams, property definitions, hubs, webhook, Slack and preflight + integrations and custom service profiles. `diff -d dir` and `apply -d dir [--dry-run]` + take such a directory as they take a team's, and apply property definitions, + environments, teams, hubs, templates, integrations and profiles in that order, before + services, experiments and schedules. Hubs are synchronized as they are applied, so + that the service profiles find the templates they name. Hubs the platform connects + itself, templates imported from a hub and Steadybit's service profiles are left out, + as every platform has them. Teams are kept without their id, which differs between + platforms, and matched by their key; `team apply` no longer sends the id, which the + platform ignores. +- Credentials of integrations are masked in exported files, and never printed by `diff`. + A masked value matches whatever the platform holds, so that an exported tenant shows no + drift and applies again: integrations that match are not applied, as the platform keeps + no secret it is not sent, and `integration ... apply` sends the stored header values and + Slack URLs in place of their masks, and leaves out a file with a masked secret that + matches the platform. +- Experiment templates, environments, teams, property definitions, hubs and integrations + have a `diff`, and their `apply` a `--dry-run`. The actions a team is given when sent + none, and the target attributes a webhook reports when sent none, are not differences. ## v6.0.1 diff --git a/README.md b/README.md index 833c38e..90cccf1 100644 --- a/README.md +++ b/README.md @@ -280,9 +280,34 @@ steadybit apply -d ./chaos --dry-run # what an apply would create or update steadybit apply -d ./chaos # profiles, services, experiments, then schedules ``` +Keep the tenant's configuration in Git the same way: experiment templates, environments, +teams, property definitions, hubs, integrations and custom service profiles, one directory +each (`templates/`, `environments/`, `teams/`, `property-definitions/`, `hubs/`, +`integrations//`, `service-profiles/`): + +```bash +steadybit export --tenant -d ./platform # needs an admin access token +steadybit diff -d ./platform +steadybit apply -d ./platform --dry-run +steadybit apply -d ./platform # definitions, environments, teams, hubs, templates, integrations, profiles +``` + +What the platform provides is left out: the hubs it connects, the templates imported from +a hub (`template import` brings them back) and Steadybit's service profiles. Hubs are +synchronized as they are applied, so that the service profiles find their templates. + +Credentials of integrations (secrets, header values, Slack webhook URLs) are written as +`'********'`. A mask stands for what the platform holds: `diff` does not report it, and +`apply` leaves out the integrations that match the platform and sends the stored header +values and URLs in place of their masks. The platform never reads a secret back, so to +change an integration that has one, put the secret in, e.g. from a CI secret, before +applying. As the platform cannot say whether that is the secret it holds, such a file is +always a difference. `diff` never prints credentials. + Each kind also has its own `diff`, and its `apply` a `--dry-run`, e.g. -`steadybit experiment diff -f ./experiments -R`. Fields the platform fills in with defaults -are not reported as differences. +`steadybit experiment diff -f ./experiments -R` or +`steadybit integration webhook diff -f ./platform/integrations/webhook -R`. Fields the +platform fills in with defaults are not reported as differences. ## In CI diff --git a/internal/cli/environment.go b/internal/cli/environment.go index cd2cb00..581d694 100644 --- a/internal/cli/environment.go +++ b/internal/cli/environment.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" "github.com/steadybit/cli/v6/internal/environment" + "github.com/steadybit/cli/v6/internal/gitops" "github.com/steadybit/cli/v6/internal/platform" "github.com/steadybit/cli/v6/internal/resource" ) @@ -48,6 +49,7 @@ func newEnvironment() *cobra.Command { RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return environment.Apply(ctx, c, a) }), } fileFlags(apply, &a.Files, &a.Recursive, "environment") + dryRun(apply, gitops.Environment, &a.Files, &a.Recursive) var d environment.DeleteOptions del := &cobra.Command{ @@ -90,6 +92,6 @@ func newEnvironment() *cobra.Command { vset.Flags().BoolVar(&vs.Replace, "replace", false, "Remove every variable not given.") variable.AddCommand(vget, vset) - cmd.AddCommand(list, get, apply, del, variable) + cmd.AddCommand(list, get, apply, newDiff(gitops.Environment, "environment", "environment.yml", "environments"), del, variable) return cmd } diff --git a/internal/cli/experiment.go b/internal/cli/experiment.go index 4ed02de..bda68c8 100644 --- a/internal/cli/experiment.go +++ b/internal/cli/experiment.go @@ -19,7 +19,7 @@ import ( func newExperiment() *cobra.Command { cmd := &cobra.Command{Use: "experiment", Short: "Check and run experiments."} cmd.AddCommand(newExperimentRun(), newExperimentGet(), newExperimentApply(), newExperimentDelete(), newExperimentDump(), newExperimentInit(), newExperimentBadge(), - newDiff(gitops.Experiment, "experiment", "experiment.yml")) + newDiff(gitops.Experiment, "experiment", "experiment.yml", "experiments")) return cmd } diff --git a/internal/cli/gitops.go b/internal/cli/gitops.go index 58b599a..4087c24 100644 --- a/internal/cli/gitops.go +++ b/internal/cli/gitops.go @@ -13,7 +13,7 @@ import ( ) // newDiff is the `diff` command of a kind of file: `experiment diff`, `schedule diff`... -func newDiff(k gitops.Kind, group, example string) *cobra.Command { +func newDiff(k gitops.Kind, group, example, dir string) *cobra.Command { var files []string var recursive bool cmd := &cobra.Command{ @@ -22,7 +22,7 @@ func newDiff(k gitops.Kind, group, example string) *cobra.Command { Args: cobra.NoArgs, Example: examples( "steadybit "+group+" diff -f "+example, - "steadybit "+group+" diff -f ./"+group+"s -R || echo drift", + "steadybit "+group+" diff -f ./"+dir+" -R || echo drift", ), RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return gitops.DiffFiles(ctx, c, k, files, recursive) @@ -57,16 +57,19 @@ func newExport() *cobra.Command { var o gitops.ExportOptions cmd := &cobra.Command{ Use: "export", - Short: "Write a team's experiments, schedules, services and the custom service profiles they use to a directory, to keep in Git.", + Short: "Write a team's experiments, schedules, services and the custom service profiles they use to a directory, to keep in Git. With --tenant, write the tenant's experiment templates, environments, teams, property definitions, hubs, integrations and custom service profiles instead.", Args: cobra.NoArgs, Example: examples( "steadybit export --team ADM -d ./chaos", + "steadybit export --tenant -d ./platform", ), RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return gitops.Export(ctx, c, o) }), } cmd.Flags().StringVar(&o.Team, "team", "", "The key of the team to export.") + cmd.Flags().BoolVar(&o.Tenant, "tenant", false, "Export the configuration of the tenant rather than a team. Credentials of integrations are written masked.") cmd.Flags().StringVarP(&o.Directory, "directory", "d", ".", "The directory to write the project to.") - _ = cmd.MarkFlagRequired("team") + cmd.MarkFlagsOneRequired("team", "tenant") + cmd.MarkFlagsMutuallyExclusive("team", "tenant") return cmd } @@ -74,11 +77,12 @@ func newApplyProject() *cobra.Command { var o gitops.ApplyOptions cmd := &cobra.Command{ Use: "apply", - Short: "Apply a project written by `export`: service profiles, then services, experiments and schedules.", + Short: "Apply a project written by `export`: property definitions, environments, teams, hubs, experiment templates, integrations and service profiles, then services, experiments and schedules.", Args: cobra.NoArgs, Example: examples( "steadybit apply -d ./chaos --dry-run", "steadybit apply -d ./chaos", + "steadybit apply -d ./platform", ), RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return gitops.ApplyProject(ctx, c, o) }), } @@ -94,7 +98,7 @@ func newDiffProject() *cobra.Command { Use: "diff", Short: "Show how a project written by `export` differs from the platform. Exits with 2 when it does.", Args: cobra.NoArgs, - Example: examples("steadybit diff -d ./chaos"), + Example: examples("steadybit diff -d ./chaos", "steadybit diff -d ./platform"), RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return gitops.DiffProject(ctx, c, dir) }), diff --git a/internal/cli/hub.go b/internal/cli/hub.go index 8d7a3a4..1405457 100644 --- a/internal/cli/hub.go +++ b/internal/cli/hub.go @@ -7,6 +7,7 @@ import ( "context" "github.com/spf13/cobra" + "github.com/steadybit/cli/v6/internal/gitops" "github.com/steadybit/cli/v6/internal/hub" "github.com/steadybit/cli/v6/internal/platform" "github.com/steadybit/cli/v6/internal/resource" @@ -45,6 +46,7 @@ func newHub() *cobra.Command { RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return hub.Apply(ctx, c, a) }), } fileFlags(apply, &a.Files, &a.Recursive, "hub") + dryRun(apply, gitops.Hub, &a.Files, &a.Recursive) apply.Flags().BoolVar(&a.Synchronize, "synchronize", false, "Fetch the hub's templates from its repository, waiting until it is done.") var d hub.DeleteOptions @@ -69,6 +71,6 @@ func newHub() *cobra.Command { } idFlag(resync, &resyncID, "The hub id.") - cmd.AddCommand(list, get, apply, del, resync) + cmd.AddCommand(list, get, apply, newDiff(gitops.Hub, "hub", "hub.yml", "hubs"), del, resync) return cmd } diff --git a/internal/cli/integration.go b/internal/cli/integration.go index 9e24b6a..b616873 100644 --- a/internal/cli/integration.go +++ b/internal/cli/integration.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/spf13/cobra" + "github.com/steadybit/cli/v6/internal/gitops" "github.com/steadybit/cli/v6/internal/integration" "github.com/steadybit/cli/v6/internal/platform" "github.com/steadybit/cli/v6/internal/resource" @@ -67,6 +68,7 @@ func newIntegrationKind(k integration.Kind) *cobra.Command { }), } fileFlags(apply, &a.Files, &a.Recursive, lower) + dryRun(apply, gitops.Integrations[k.Name], &a.Files, &a.Recursive) var d integration.DeleteOptions del := &cobra.Command{ @@ -81,6 +83,6 @@ func newIntegrationKind(k integration.Kind) *cobra.Command { idFlag(del, &d.ID, "The "+lower+" id.") del.Flags().BoolVar(&d.Yes, "yes", false, yesHelp) - cmd.AddCommand(list, get, apply, del) + cmd.AddCommand(list, get, apply, newDiff(gitops.Integrations[k.Name], "integration "+k.Name, k.Name+".yml", "integrations/"+k.Name), del) return cmd } diff --git a/internal/cli/property.go b/internal/cli/property.go index 2a63df7..155a46f 100644 --- a/internal/cli/property.go +++ b/internal/cli/property.go @@ -7,6 +7,7 @@ import ( "context" "github.com/spf13/cobra" + "github.com/steadybit/cli/v6/internal/gitops" "github.com/steadybit/cli/v6/internal/platform" "github.com/steadybit/cli/v6/internal/property" "github.com/steadybit/cli/v6/internal/resource" @@ -65,6 +66,7 @@ func newPropertyDefinition() *cobra.Command { } fileFlags(apply, &a.Files, &a.Recursive, "property definition") apply.Flags().BoolVar(&a.DeleteValues, "delete-values", false, "Allow removing enum values still in use, deleting them where they are used.") + dryRun(apply, gitops.PropertyDefinition, &a.Files, &a.Recursive) var d property.DeleteDefinitionOptions del := &cobra.Command{ @@ -80,7 +82,7 @@ func newPropertyDefinition() *cobra.Command { del.Flags().BoolVar(&d.Associations, "delete-associations", false, "Also delete the associations of the property.") del.Flags().BoolVar(&d.Yes, "yes", false, yesHelp) - cmd.AddCommand(list, get, apply, del) + cmd.AddCommand(list, get, apply, newDiff(gitops.PropertyDefinition, "property definition", "result-color.yml", "property-definitions"), del) return cmd } diff --git a/internal/cli/schedule.go b/internal/cli/schedule.go index 958b1af..23f5ad4 100644 --- a/internal/cli/schedule.go +++ b/internal/cli/schedule.go @@ -126,7 +126,7 @@ func newSchedule() *cobra.Command { scheduleIDFlag(c, &id) return c } - cmd.AddCommand(list, get, apply, newDiff(gitops.Schedule, "schedule", "schedule.yml"), create, update, + cmd.AddCommand(list, get, apply, newDiff(gitops.Schedule, "schedule", "schedule.yml", "schedules"), create, update, idCommand("enable", "Enable an experiment schedule.", func(ctx context.Context, c *platform.Client, id string) error { return schedule.SetEnabled(ctx, c, id, true) }), diff --git a/internal/cli/service.go b/internal/cli/service.go index 1bc010f..46cadde 100644 --- a/internal/cli/service.go +++ b/internal/cli/service.go @@ -198,7 +198,7 @@ func newService() *cobra.Command { vset.Flags().BoolVar(&vs.Replace, "replace", false, "Remove every variable not given.") variable.AddCommand(vget, vset) - cmd.AddCommand(list, get, apply, newDiff(gitops.Service, "service", "service.yml"), del, risk, experiments, variable) + cmd.AddCommand(list, get, apply, newDiff(gitops.Service, "service", "service.yml", "services"), del, risk, experiments, variable) return cmd } @@ -260,6 +260,6 @@ func newServiceProfile() *cobra.Command { } idFlag(del, &deleteID, "The service profile id.") - cmd.AddCommand(list, get, apply, newDiff(gitops.ServiceProfile, "service-profile", "profile.yml"), del) + cmd.AddCommand(list, get, apply, newDiff(gitops.ServiceProfile, "service-profile", "profile.yml", "service-profiles"), del) return cmd } diff --git a/internal/cli/team.go b/internal/cli/team.go index cfc63a2..3f4f40b 100644 --- a/internal/cli/team.go +++ b/internal/cli/team.go @@ -7,6 +7,7 @@ import ( "context" "github.com/spf13/cobra" + "github.com/steadybit/cli/v6/internal/gitops" "github.com/steadybit/cli/v6/internal/platform" "github.com/steadybit/cli/v6/internal/resource" "github.com/steadybit/cli/v6/internal/team" @@ -51,6 +52,7 @@ func newTeam() *cobra.Command { RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return team.Apply(ctx, c, a) }), } fileFlags(apply, &a.Files, &a.Recursive, "team") + dryRun(apply, gitops.Team, &a.Files, &a.Recursive) var d team.DeleteOptions del := &cobra.Command{ @@ -64,7 +66,7 @@ func newTeam() *cobra.Command { del.Flags().BoolVar(&d.Experiments, "purge-experiments", false, "Also delete the team's experiments and their runs.") del.Flags().BoolVar(&d.Yes, "yes", false, yesHelp) - cmd.AddCommand(list, get, apply, del, newTeamMember(), newTeamEnvironment()) + cmd.AddCommand(list, get, apply, newDiff(gitops.Team, "team", "team.yml", "teams"), del, newTeamMember(), newTeamEnvironment()) return cmd } diff --git a/internal/cli/template.go b/internal/cli/template.go index 5d28dbe..ebd3a0b 100644 --- a/internal/cli/template.go +++ b/internal/cli/template.go @@ -7,6 +7,7 @@ import ( "context" "github.com/spf13/cobra" + "github.com/steadybit/cli/v6/internal/gitops" "github.com/steadybit/cli/v6/internal/platform" "github.com/steadybit/cli/v6/internal/resource" "github.com/steadybit/cli/v6/internal/template" @@ -80,6 +81,7 @@ func newTemplate() *cobra.Command { RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return template.Apply(ctx, c, a) }), } fileFlags(apply, &a.Files, &a.Recursive, "template") + dryRun(apply, gitops.Template, &a.Files, &a.Recursive) var d template.DeleteOptions del := &cobra.Command{ @@ -107,6 +109,6 @@ func newTemplate() *cobra.Command { _ = imp.MarkFlagRequired("template") variadic(imp, "template") - cmd.AddCommand(list, get, apply, del, imp) + cmd.AddCommand(list, get, apply, newDiff(gitops.Template, "template", "template.yml", "templates"), del, imp) return cmd } diff --git a/internal/environment/environment.go b/internal/environment/environment.go index d9574ac..36f687e 100644 --- a/internal/environment/environment.go +++ b/internal/environment/environment.go @@ -19,7 +19,7 @@ import ( // The state is the platform's, and the version is dropped as `service get` drops it: kept // in a file, it turns every apply after an edit in the UI into a conflict. -var readOnly = []string{"version", "state"} +var ReadOnly = []string{"version", "state"} func uuid(id string) (openapi_types.UUID, error) { u, ok := resource.UUID(id) @@ -88,7 +88,7 @@ func Get(ctx context.Context, c *platform.Client, o GetOptions) error { if err != nil { return notFoundOr(err, o.ID, "Failed to get environment %s") } - if err := resource.Output(resource.Strip(doc, readOnly...), o.File, o.Type); err != nil { + if err := resource.Output(resource.Strip(doc, ReadOnly...), o.File, o.Type); err != nil { return err } if o.File != "" { @@ -109,7 +109,7 @@ func Apply(ctx context.Context, c *platform.Client, o ApplyOptions) error { return resource.Applied{}, fmt.Errorf("Environment file '%s' does not name the environment.", file) } var saved struct{ ID, Name string } - resp, err := c.UpsertEnvironmentWithBody(ctx, "application/json", resource.Body(resource.Strip(doc, readOnly...).Value())) + resp, err := c.UpsertEnvironmentWithBody(ctx, "application/json", resource.Body(resource.Strip(doc, ReadOnly...).Value())) resp, err = platform.Decode(resp, err, &saved) if err != nil { return resource.Applied{}, platform.Failed(err, "Failed to save environment %s", name) diff --git a/internal/gitops/kinds.go b/internal/gitops/kinds.go index c90ec21..c4c18ca 100644 --- a/internal/gitops/kinds.go +++ b/internal/gitops/kinds.go @@ -7,24 +7,35 @@ import ( "context" "fmt" "net/http" + "strings" openapi_types "github.com/oapi-codegen/runtime/types" "github.com/steadybit/cli/v6/api" + "github.com/steadybit/cli/v6/internal/environment" + "github.com/steadybit/cli/v6/internal/hub" + "github.com/steadybit/cli/v6/internal/integration" "github.com/steadybit/cli/v6/internal/jsyaml" "github.com/steadybit/cli/v6/internal/output" "github.com/steadybit/cli/v6/internal/platform" + "github.com/steadybit/cli/v6/internal/property" + "github.com/steadybit/cli/v6/internal/team" + "github.com/steadybit/cli/v6/internal/template" ) // Kind is a type of file kept in Git and how to find its counterpart on the platform. type Kind struct { // Name is what messages call it, "experiment" or "service profile". Name string - // ReadOnly fields are reported by the platform but not part of the file. + // ReadOnly fields are reported by the platform but not part of the file. In + // "members.name", the field is dropped from each member. ReadOnly []string - // Defaults are the values the platform gives fields a file leaves out. Holding - // exactly that, such a field is not a difference; holding anything else, applying - // the file would reset it, which is. + // Defaults are the values the platform gives fields a file leaves out, or leaves as + // an empty list or map. Holding exactly that, such a field is not a difference; + // holding anything else, applying the file would reset it, which is. Defaults map[string]any + // Secrets are fields holding credentials, as integration.Kind names them. A mask in + // a file stands for whatever the platform holds, and no value is ever printed. + Secrets []string // Identity is the field that names it on the platform. A file matched otherwise, by // an externalId or a name, has none yet, which is not a difference. Identity string @@ -157,14 +168,164 @@ var ServiceProfile = Kind{ }, } +// byID finds the platform's version of a file by the id in it. A file without one is new; +// one whose id is no UUID names nothing on the platform, and applying it would not work. +func byID(get func(ctx context.Context, c *platform.Client, id openapi_types.UUID) (*http.Response, error)) func(context.Context, *platform.Client, *jsyaml.Map) (string, *jsyaml.Map, error) { + return func(ctx context.Context, c *platform.Client, local *jsyaml.Map) (string, *jsyaml.Map, error) { + id := str(local, "id") + if id == "" { + return "", nil, nil + } + u, ok := uuid(id) + if !ok { + return "", nil, fmt.Errorf("%s is not a valid id", id) + } + remote, err := fetch(get(ctx, c, u)) + return id, remote, err + } +} + +var Template = Kind{ + Name: "experiment template", + ReadOnly: template.ReadOnly, + Identity: "id", + Remote: byID(func(ctx context.Context, c *platform.Client, id openapi_types.UUID) (*http.Response, error) { + return c.GetExperimentTemplate(ctx, id) + }), +} + +var Environment = Kind{ + Name: "environment", + ReadOnly: environment.ReadOnly, + Identity: "id", + // By its id, or by its name, which the platform matches a file without an id by. + Remote: func(ctx context.Context, c *platform.Client, local *jsyaml.Map) (string, *jsyaml.Map, error) { + if id := str(local, "id"); id != "" { + return byID(func(ctx context.Context, c *platform.Client, id openapi_types.UUID) (*http.Response, error) { + return c.GetEnvironment(ctx, id) + })(ctx, c, local) + } + name := str(local, "name") + if name == "" { + return "", nil, nil + } + var list struct { + Environments []struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"environments"` + } + resp, err := c.GetEnvironments(ctx, &api.GetEnvironmentsParams{Search: &name}) + if _, err := platform.Decode(resp, err, &list); err != nil { + return "", nil, err + } + for _, e := range list.Environments { + if e.Name == name { + u, _ := uuid(e.ID) + remote, err := fetch(c.GetEnvironment(ctx, u)) + return e.ID, remote, err + } + } + return "", nil, nil + }, +} + +// Team is matched by its key, as the platform upserts it. The id differs from one platform +// to the next, so it is neither exported nor compared: a team kept in Git matches its +// namesake anywhere. +var Team = Kind{ + Name: "team", + ReadOnly: append(append([]string{"id"}, team.ReadOnly...), prefixed("members.", team.MemberReadOnly)...), + // Sent none, a team may still wait and validate services. + Defaults: map[string]any{"allowedActions": []any{"wait", "service-validation"}, "managedBy": "MANUAL"}, + Identity: "key", + Remote: func(ctx context.Context, c *platform.Client, local *jsyaml.Map) (string, *jsyaml.Map, error) { + key := str(local, "key") + if key == "" { + return "", nil, nil + } + remote, err := fetch(c.GetTeam(ctx, key)) + return key, remote, err + }, +} + +var PropertyDefinition = Kind{ + Name: "property definition", + ReadOnly: property.ReadOnly, + Identity: "key", + Remote: func(ctx context.Context, c *platform.Client, local *jsyaml.Map) (string, *jsyaml.Map, error) { + key := str(local, "key") + if key == "" { + return "", nil, nil + } + remote, err := fetch(c.GetPropertyDefinition(ctx, key)) + return key, remote, err + }, +} + +var Hub = Kind{ + Name: "hub", + ReadOnly: hub.ReadOnly, + Identity: "id", + Remote: byID(func(ctx context.Context, c *platform.Client, id openapi_types.UUID) (*http.Response, error) { + return c.GetHubById(ctx, id) + }), +} + +// Integrations are the kinds of integration by their command's name. +var Integrations = map[string]Kind{ + integration.Webhook.Name: integrationKind(integration.Webhook, "webhook integration", allTargetAttributes), + integration.Slack.Name: integrationKind(integration.Slack, "Slack integration", nil), + integration.Preflight.Name: integrationKind(integration.Preflight, "preflight webhook", allTargetAttributes), + integration.PreflightAction.Name: integrationKind(integration.PreflightAction, "preflight action integration", nil), +} + +// Sent none, a webhook reports every target attribute. +var allTargetAttributes = map[string]any{"targetAttributeIncludes": []any{"*"}} + +func integrationKind(k integration.Kind, name string, defaults map[string]any) Kind { + return Kind{ + Name: name, + ReadOnly: integration.ReadOnly, + Defaults: defaults, + Secrets: k.Secrets, + Identity: "id", + Remote: byID(k.Fetch), + } +} + +func prefixed(prefix string, fields []string) []string { + out := make([]string, len(fields)) + for i, f := range fields { + out[i] = prefix + f + } + return out +} + func strip(m *jsyaml.Map, fields []string) *jsyaml.Map { c := jsyaml.Clone(m).(*jsyaml.Map) for _, f := range fields { - c.Delete(f) + deletePath(c, strings.Split(f, ".")) } return c } +// deletePath removes a field, going into every item of the lists on its way. +func deletePath(v any, path []string) { + switch x := v.(type) { + case *jsyaml.Map: + if len(path) == 1 { + x.Delete(path[0]) + } else if child, ok := x.Get(path[0]); ok { + deletePath(child, path[1:]) + } + case []any: + for _, item := range x { + deletePath(item, path) + } + } +} + // State is what applying a file would do. type State int @@ -200,7 +361,9 @@ func Compare(ctx context.Context, c *platform.Client, k Kind, file string, local for key := range k.Defaults { defaulted = append(defaulted, key) } - diff, err := Diff(file, strip(local.Value(), ignored), withoutDefaults(local.Value(), strip(remote, ignored), k.Defaults), defaulted...) + l, r := strip(local.Value(), ignored), withoutDefaults(local.Value(), strip(remote, ignored), k.Defaults) + hideSecrets(l, r, k.Secrets) + diff, err := Diff(file, l, r, defaulted...) if err != nil { return Result{}, err } @@ -221,16 +384,80 @@ func (r Result) Describe(k Kind) string { return fmt.Sprintf("%s matches %s %s.", r.File, k.Name, r.ID) } -// withoutDefaults drops the fields the file leaves out that hold the platform's default. +// withoutDefaults drops the fields the file leaves out that hold the platform's default, +// and takes the file's empty list or map for the default the platform turned it into. func withoutDefaults(local, remote *jsyaml.Map, defaults map[string]any) *jsyaml.Map { out := jsyaml.Clone(remote).(*jsyaml.Map) for key, value := range defaults { - if _, set := local.Get(key); set { + held, ok := out.Get(key) + if !ok || jsyaml.CompactJSON(held) != jsyaml.CompactJSON(value) { continue } - if held, ok := out.Get(key); ok && jsyaml.CompactJSON(held) == jsyaml.CompactJSON(value) { + switch set, has := local.Get(key); { + case !has: out.Delete(key) + case isEmptyCollection(set): + out.Set(key, set) } } return out } + +func isEmptyCollection(v any) bool { + switch x := v.(type) { + case []any: + return len(x) == 0 + case *jsyaml.Map: + return x.Len() == 0 + } + return false +} + +// hideSecrets makes credentials comparable without printing them. The platform's value +// shows as the mask, which a mask in the file matches, as does the value itself; any +// other value in the file is what applying it would set, and shows as that. +func hideSecrets(local, remote *jsyaml.Map, secrets []string) { + for _, field := range secrets { + lv, _ := local.Get(field) + rv, _ := remote.Get(field) + lm, lIsMap := lv.(*jsyaml.Map) + rm, rIsMap := rv.(*jsyaml.Map) + if lIsMap || rIsMap { + if lm == nil { + lm = jsyaml.NewMap() + } + if rm == nil { + rm = jsyaml.NewMap() + } + for _, key := range lm.Keys() { + hideSecret(lm, rm, key) + } + for _, key := range rm.Keys() { + hideSecret(lm, rm, key) + } + continue + } + hideSecret(local, remote, field) + } +} + +// hideSecret masks any value a secret field holds, whatever its type: an unquoted number +// in a file is as much a credential as a string. +func hideSecret(local, remote *jsyaml.Map, key string) { + lv, lok := local.Get(key) + rv, rok := remote.Get(key) + if rok && !blank(rv) { + remote.Set(key, integration.Mask) + } + switch { + case !lok || blank(lv) || lv == integration.Mask: + case integration.Masked(lv) || (rok && jsyaml.CompactJSON(lv) == jsyaml.CompactJSON(rv)): + local.Set(key, integration.Mask) + default: + local.Set(key, integration.Mask+" (from the file)") + } +} + +func blank(v any) bool { + return v == nil || v == "" +} diff --git a/internal/gitops/project.go b/internal/gitops/project.go index 83126b8..7261845 100644 --- a/internal/gitops/project.go +++ b/internal/gitops/project.go @@ -15,26 +15,90 @@ import ( "strings" "github.com/steadybit/cli/v6/api" + "github.com/steadybit/cli/v6/internal/environment" "github.com/steadybit/cli/v6/internal/experiment" + "github.com/steadybit/cli/v6/internal/hub" + "github.com/steadybit/cli/v6/internal/integration" "github.com/steadybit/cli/v6/internal/jsyaml" "github.com/steadybit/cli/v6/internal/output" "github.com/steadybit/cli/v6/internal/platform" + "github.com/steadybit/cli/v6/internal/property" "github.com/steadybit/cli/v6/internal/schedule" "github.com/steadybit/cli/v6/internal/service" "github.com/steadybit/cli/v6/internal/serviceprofile" + "github.com/steadybit/cli/v6/internal/team" + "github.com/steadybit/cli/v6/internal/template" ) -// A project is a directory holding what a team keeps in Git, one kind per directory, -// in the order applying them has to follow: services name their profile, schedules -// their experiment. +// A project is a directory holding what a team or the tenant keeps in Git, one kind per +// directory, in the order applying them has to follow. Templates and experiments carry +// the properties definitions define; teams name their environments, integrations their +// team, service profiles their templates, services their profile, experiments their +// team and environment, schedules their experiment. Hubs depend on nothing. var projectKinds = []struct { - dir string - kind Kind + dir string + kind Kind + apply func(ctx context.Context, c *platform.Client, path string, o ApplyOptions) error }{ - {"service-profiles", ServiceProfile}, - {"services", Service}, - {"experiments", Experiment}, - {"schedules", Schedule}, + {"property-definitions", PropertyDefinition, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + return property.ApplyDefinitions(ctx, c, property.ApplyDefinitionOptions{Files: []string{path}, Recursive: true}) + }}, + {"environments", Environment, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + return environment.Apply(ctx, c, environment.ApplyOptions{Files: []string{path}, Recursive: true}) + }}, + {"teams", Team, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + return team.Apply(ctx, c, team.ApplyOptions{Files: []string{path}, Recursive: true}) + }}, + // Synchronized, as `hub apply --synchronize` does: service profiles name the templates + // a hub brings, so on a restored tenant those have to be there first. A hub that cannot + // be synchronized ends the apply. + {"hubs", Hub, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + return hub.Apply(ctx, c, hub.ApplyOptions{Files: []string{path}, Recursive: true, Synchronize: true}) + }}, + {"templates", Template, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + return template.Apply(ctx, c, template.ApplyOptions{Files: []string{path}, Recursive: true}) + }}, + {"integrations/webhook", Integrations[integration.Webhook.Name], applyIntegrations(integration.Webhook)}, + {"integrations/slack", Integrations[integration.Slack.Name], applyIntegrations(integration.Slack)}, + {"integrations/preflight", Integrations[integration.Preflight.Name], applyIntegrations(integration.Preflight)}, + {"integrations/preflight-action", Integrations[integration.PreflightAction.Name], applyIntegrations(integration.PreflightAction)}, + {"service-profiles", ServiceProfile, func(ctx context.Context, c *platform.Client, path string, o ApplyOptions) error { + return serviceprofile.Apply(ctx, c, serviceprofile.ApplyOptions{Files: []string{path}, Recursive: true, DeleteExperiments: o.DeleteExperiments}) + }}, + {"services", Service, func(ctx context.Context, c *platform.Client, path string, o ApplyOptions) error { + return service.Apply(ctx, c, service.ApplyOptions{Files: []string{path}, Recursive: true, DeleteExperiments: o.DeleteExperiments}) + }}, + {"experiments", Experiment, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + return experiment.Apply(ctx, c, experiment.ApplyOptions{Files: []string{path}, Recursive: true}) + }}, + {"schedules", Schedule, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + return schedule.Apply(ctx, c, schedule.ApplyOptions{Files: []string{path}, Recursive: true}) + }}, +} + +// applyIntegrations leaves out the files that match the platform. Those holding a masked +// secret could not be applied, the platform keeping no secret it is not sent, and the +// others need not be. +func applyIntegrations(k integration.Kind) func(ctx context.Context, c *platform.Client, path string, o ApplyOptions) error { + return func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + gk := Integrations[k.Name] + results, err := compareAll(ctx, c, gk, []string{path}, true) + if err != nil { + return err + } + var changed []string + for _, r := range results { + if r.State == Unchanged { + fmt.Println(r.Describe(gk)) + } else { + changed = append(changed, r.File) + } + } + if len(changed) == 0 { + return nil + } + return integration.Apply(ctx, c, k, integration.ApplyOptions{Files: changed}) + } } var unsafe = regexp.MustCompile(`[^a-z0-9._-]+`) @@ -63,12 +127,17 @@ func writeDocument(file string, doc *jsyaml.Map, readOnly []string) error { type ExportOptions struct { Directory string Team string + Tenant bool } // Export writes a team's experiments, schedules and services, and the custom service -// profiles those services use, as a project. Files are only written, never removed, so -// something deleted on the platform keeps its file until it is removed by hand. +// profiles those services use, as a project; or the tenant's configuration. Files are +// only written, never removed, so something deleted on the platform keeps its file until +// it is removed by hand. func Export(ctx context.Context, c *platform.Client, o ExportOptions) error { + if o.Tenant { + return exportTenant(ctx, c, o) + } taken := map[string]bool{} counts := map[string]int{} team := []string{o.Team} @@ -169,14 +238,16 @@ func mapWith(key, value string) *jsyaml.Map { // projectDirs are the kinds a project directory holds, in the order to apply them. func projectDirs(dir string) (map[string]string, error) { found := map[string]string{} + var names []string for _, pk := range projectKinds { - path := filepath.Join(dir, pk.dir) + names = append(names, pk.dir+"/") + path := filepath.Join(dir, filepath.FromSlash(pk.dir)) if info, err := os.Stat(path); err == nil && info.IsDir() { found[pk.dir] = path } } if len(found) == 0 { - return nil, fmt.Errorf("'%s' holds none of experiments/, schedules/, services/ or service-profiles/.", dir) + return nil, fmt.Errorf("'%s' holds none of %s or %s.", dir, strings.Join(names[:len(names)-1], ", "), names[len(names)-1]) } return found, nil } @@ -187,7 +258,7 @@ type ApplyOptions struct { DryRun bool } -// ApplyProject applies every kind in a project, profiles first and schedules last. +// ApplyProject applies every kind in a project, in the order of projectKinds. func ApplyProject(ctx context.Context, c *platform.Client, o ApplyOptions) error { dirs, err := projectDirs(o.Directory) if err != nil { @@ -201,16 +272,7 @@ func ApplyProject(ctx context.Context, c *platform.Client, o ApplyOptions) error if o.DryRun { err = DryRun(ctx, c, pk.kind, []string{path}, true) } else { - switch pk.dir { - case "service-profiles": - err = serviceprofile.Apply(ctx, c, serviceprofile.ApplyOptions{Files: []string{path}, Recursive: true, DeleteExperiments: o.DeleteExperiments}) - case "services": - err = service.Apply(ctx, c, service.ApplyOptions{Files: []string{path}, Recursive: true, DeleteExperiments: o.DeleteExperiments}) - case "experiments": - err = experiment.Apply(ctx, c, experiment.ApplyOptions{Files: []string{path}, Recursive: true}) - case "schedules": - err = schedule.Apply(ctx, c, schedule.ApplyOptions{Files: []string{path}, Recursive: true}) - } + err = pk.apply(ctx, c, path, o) } if err != nil { return err diff --git a/internal/gitops/project_test.go b/internal/gitops/project_test.go index 9eb2c6c..4f9243c 100644 --- a/internal/gitops/project_test.go +++ b/internal/gitops/project_test.go @@ -79,5 +79,5 @@ func TestAProjectEditedLocallyDrifts(t *testing.T) { func TestApplyNeedsAProject(t *testing.T) { err := gitops.ApplyProject(ctx, nil, gitops.ApplyOptions{Directory: t.TempDir()}) - assert.ErrorContains(t, err, "holds none of experiments/, schedules/, services/ or service-profiles/.") + assert.ErrorContains(t, err, "holds none of property-definitions/, environments/, teams/, hubs/, templates/, integrations/webhook/, integrations/slack/, integrations/preflight/, integrations/preflight-action/, service-profiles/, services/, experiments/ or schedules/.") } diff --git a/internal/gitops/tenant.go b/internal/gitops/tenant.go new file mode 100644 index 0000000..0c8b923 --- /dev/null +++ b/internal/gitops/tenant.go @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2026 Steadybit GmbH + +package gitops + +import ( + "context" + "errors" + "fmt" + "net/http" + "path/filepath" + + openapi_types "github.com/oapi-codegen/runtime/types" + "github.com/steadybit/cli/v6/api" + "github.com/steadybit/cli/v6/internal/integration" + "github.com/steadybit/cli/v6/internal/jsyaml" + "github.com/steadybit/cli/v6/internal/platform" +) + +// The hubs every tenant is connected to by the platform itself, with the same ids +// everywhere. Kept in Git, they would be connected a second time on another platform. +var providedHubs = map[string]bool{ + "6a55640d-72d4-4194-a058-0afcf731dfac": true, // Steadybit Reliability Hub + "738c90be-bdc4-4e0b-a4de-c8f1144b25c7": true, // Steadybit Service Templates +} + +// tenantExport writes one kind of the tenant's configuration. +type tenantExport struct { + o ExportOptions + taken map[string]bool + counts map[string]int +} + +func (e *tenantExport) write(dir, name string, doc *jsyaml.Map, k Kind) error { + e.counts[dir]++ + return writeDocument(fileName(filepath.Join(e.o.Directory, dir), name, e.taken), doc, k.ReadOnly) +} + +func (e *tenantExport) each(ids []string, what string, get func(id string) (*http.Response, error), write func(doc *jsyaml.Map) error) error { + for _, id := range ids { + doc, _, err := platform.ReadDocument(get(id)) + if err != nil { + return platform.Failed(err, "Failed to get %s %s", what, id) + } + if err := write(doc.Value()); err != nil { + return err + } + } + return nil +} + +func byUUID(get func(openapi_types.UUID) (*http.Response, error)) func(string) (*http.Response, error) { + return func(id string) (*http.Response, error) { + u, ok := uuid(id) + if !ok { + return nil, errors.New("not a valid id") + } + return get(u) + } +} + +// exportTenant writes what the tenant's admins configure. What the platform provides +// is left out, as it comes with every platform: the hubs it connects, its service +// profiles, and the templates imported from a hub, which importing again brings back. +func exportTenant(ctx context.Context, c *platform.Client, o ExportOptions) error { + e := &tenantExport{o: o, taken: map[string]bool{}, counts: map[string]int{}} + + type keyed struct { + Key string `json:"key"` + } + definitions, err := platform.AllPages[keyed](func(page, size int32) (*http.Response, error) { + return c.GetPropertyDefinitions(ctx, &api.GetPropertyDefinitionsParams{Page: api.PageRequestAO{Page: &page, Size: &size}}) + }) + if err != nil { + return platform.Failed(err, "Failed to get the property definitions") + } + var keys []string + for _, d := range definitions { + keys = append(keys, d.Key) + } + if err := e.each(keys, "property definition", func(key string) (*http.Response, error) { return c.GetPropertyDefinition(ctx, key) }, + func(doc *jsyaml.Map) error { + return e.write("property-definitions", str(doc, "key"), doc, PropertyDefinition) + }); err != nil { + return err + } + + var environments struct { + Environments []struct { + ID string `json:"id"` + } `json:"environments"` + } + resp, err := c.GetEnvironments(ctx, &api.GetEnvironmentsParams{}) + if _, err := platform.Decode(resp, err, &environments); err != nil { + return platform.Failed(err, "Failed to get the environments") + } + var ids []string + for _, env := range environments.Environments { + ids = append(ids, env.ID) + } + if err := e.each(ids, "environment", byUUID(func(id openapi_types.UUID) (*http.Response, error) { return c.GetEnvironment(ctx, id) }), + func(doc *jsyaml.Map) error { return e.write("environments", str(doc, "name"), doc, Environment) }); err != nil { + return err + } + + var teams struct { + Teams []keyed `json:"teams"` + } + resp, err = c.GetTeams(ctx, &api.GetTeamsParams{}) + if _, err := platform.Decode(resp, err, &teams); err != nil { + return platform.Failed(err, "Failed to get the teams") + } + keys = nil + for _, t := range teams.Teams { + keys = append(keys, t.Key) + } + if err := e.each(keys, "team", func(key string) (*http.Response, error) { return c.GetTeam(ctx, key) }, + func(doc *jsyaml.Map) error { return e.write("teams", str(doc, "key"), doc, Team) }); err != nil { + return err + } + + var hubs struct { + Hubs []struct { + ID string `json:"id"` + } `json:"hubs"` + } + resp, err = c.GetHubs(ctx) + if _, err := platform.Decode(resp, err, &hubs); err != nil { + return platform.Failed(err, "Failed to get the hubs") + } + ids = nil + for _, h := range hubs.Hubs { + ids = append(ids, h.ID) + } + // An imported template keeps the id it has in its hub, which is how one is told. + imported := map[string]bool{} + if err := e.each(ids, "hub", byUUID(func(id openapi_types.UUID) (*http.Response, error) { return c.GetHubById(ctx, id) }), + func(doc *jsyaml.Map) error { + templates, _ := doc.Get("templates") + list, _ := templates.([]any) + for _, t := range list { + if m, ok := t.(*jsyaml.Map); ok { + imported[str(m, "id")] = true + } + } + if providedHubs[str(doc, "id")] { + return nil + } + return e.write("hubs", str(doc, "hubName"), doc, Hub) + }); err != nil { + return err + } + + var templates struct { + Templates []struct { + ID string `json:"id"` + } `json:"templates"` + } + // Hidden templates, and those whose actions, target types or property definitions are + // not available right now, are the tenant's too; the platform lists them only if asked. + all := true + resp, err = c.GetExperimentTemplates(ctx, &api.GetExperimentTemplatesParams{IncludeHidden: &all, IncludeNonAvailable: &all}) + if _, err := platform.Decode(resp, err, &templates); err != nil { + return platform.Failed(err, "Failed to get the experiment templates") + } + ids = nil + for _, t := range templates.Templates { + if !imported[t.ID] { + ids = append(ids, t.ID) + } + } + if err := e.each(ids, "experiment template", byUUID(func(id openapi_types.UUID) (*http.Response, error) { return c.GetExperimentTemplate(ctx, id) }), + func(doc *jsyaml.Map) error { return e.write("templates", str(doc, "templateTitle"), doc, Template) }); err != nil { + return err + } + + for _, k := range integration.Kinds { + var list struct { + Content []struct { + ID string `json:"id"` + } `json:"content"` + } + resp, err := k.FetchAll(ctx, c) + if _, err := platform.Decode(resp, err, &list); err != nil { + return platform.Failed(err, "Failed to get the %s", k.Plural) + } + ids = nil + for _, i := range list.Content { + ids = append(ids, i.ID) + } + dir := "integrations/" + k.Name + if err := e.each(ids, k.Title, byUUID(func(id openapi_types.UUID) (*http.Response, error) { return k.Fetch(ctx, c, id) }), + func(doc *jsyaml.Map) error { + k.MaskSecrets(doc) + e.counts["integrations"]++ + return e.write(dir, str(doc, "name"), doc, Integrations[k.Name]) + }); err != nil { + return err + } + } + + // Profiles Steadybit provides come with the platform. + type profile struct { + ID string `json:"id"` + Origin string `json:"origin"` + } + profiles, err := platform.AllPages[profile](func(page, size int32) (*http.Response, error) { + return c.GetProfiles(ctx, &api.GetProfilesParams{Page: api.PageRequestAO{Page: &page, Size: &size}}) + }) + if err != nil { + return platform.Failed(err, "Failed to get the service profiles") + } + ids = nil + for _, p := range profiles { + if p.Origin == "CUSTOM" { + ids = append(ids, p.ID) + } + } + if err := e.each(ids, "service profile", byUUID(func(id openapi_types.UUID) (*http.Response, error) { return c.GetProfile(ctx, id) }), + func(doc *jsyaml.Map) error { return e.write("service-profiles", str(doc, "name"), doc, ServiceProfile) }); err != nil { + return err + } + + fmt.Printf("Exported the tenant to %s: %d experiment templates, %d environments, %d teams, %d property definitions, %d hubs, %d integrations, %d service profiles.\n", + o.Directory, e.counts["templates"], e.counts["environments"], e.counts["teams"], e.counts["property-definitions"], e.counts["hubs"], + e.counts["integrations"], e.counts["service-profiles"]) + return nil +} diff --git a/internal/gitops/tenant_test.go b/internal/gitops/tenant_test.go new file mode 100644 index 0000000..20af628 --- /dev/null +++ b/internal/gitops/tenant_test.go @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2026 Steadybit GmbH + +package gitops_test + +import ( + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steadybit/cli/v6/internal/gitops" + "github.com/steadybit/cli/v6/internal/platformtest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + environmentID = "019eacd7-fb2c-733a-bed5-99a935323d01" + teamID = "019eacd7-fb2c-733a-bed5-99a935323d02" + hubID = "019eacd7-fb2c-733a-bed5-99a935323d03" + templateID = "019eacd7-fb2c-733a-bed5-99a935323d04" + importedID = "019eacd7-fb2c-733a-bed5-99a935323d05" + webhookID = "019eacd7-fb2c-733a-bed5-99a935323d06" + slackID = "019eacd7-fb2c-733a-bed5-99a935323d07" + providedHubID = "6a55640d-72d4-4194-a058-0afcf731dfac" +) + +const ( + storedWebhook = `{"id":"` + webhookID + `","version":0,"scope":"GLOBAL","name":"Notify","url":"https://example.com/hook","secret":"****************","events":["*"],"targetAttributeIncludes":["*"],"headers":{"Authorization":"Bearer abc"}}` + storedSlack = `{"id":"` + slackID + `","version":1,"scope":"GLOBAL","name":"Chat","url":"https://hooks.example.com/T1/B2/xyz","events":["*"],"channel":"#chaos"}` +) + +func fakeTenant(t *testing.T) *platformtest.Platform { + p := platformtest.New(t) + p.Reply("GET /api/properties/definitions", platformtest.Reply{JSON: map[string]any{"items": []any{map[string]any{"key": "tribe"}}}}) + p.Reply("GET /api/properties/definitions/tribe", platformtest.Reply{Body: `{"key":"tribe","label":"Tribe","dataType":"STRING","version":3}`}) + p.Reply("GET /api/environments", platformtest.Reply{JSON: map[string]any{"environments": []any{map[string]any{"id": environmentID}}}}) + p.Reply("GET /api/environments/"+environmentID, platformtest.Reply{Body: `{"id":"` + environmentID + `","name":"Prod","version":2,"predicate":{"operator":"AND","predicates":[]},"state":"READY"}`}) + p.Reply("GET /api/teams", platformtest.Reply{JSON: map[string]any{"teams": []any{map[string]any{"key": "ADM"}}}}) + p.Reply("GET /api/teams/ADM", platformtest.Reply{Body: `{"id":"` + teamID + `","key":"ADM","name":"Admins","allowedActions":["wait","service-validation"],"allowedEnvironments":["Prod"],"managedBy":"MANUAL","version":0,` + + `"members":[{"username":"u1","name":"Jane","email":"jane@example.com","role":"OWNER","pictureUrl":"https://example.com/jane.png","managedBy":"MANUAL"}]}`}) + p.Reply("GET /api/hubs", platformtest.Reply{JSON: map[string]any{"hubs": []any{map[string]any{"id": providedHubID}, map[string]any{"id": hubID}}}}) + p.Reply("GET /api/hubs/"+providedHubID, platformtest.Reply{Body: `{"hubName":"Steadybit Reliability Hub","id":"` + providedHubID + `","templates":[{"id":"` + importedID + `","templateTitle":"Imported"}]}`}) + p.Reply("GET /api/hubs/"+hubID, platformtest.Reply{Body: `{"hubName":"Own Hub","repositoryUrl":"https://example.com/index.json","id":"` + hubID + `","version":4,"templates":[],"lastSync":"x","created":"c"}`}) + p.Reply("GET /api/experiments/templates", platformtest.Reply{JSON: map[string]any{"templates": []any{map[string]any{"id": templateID}, map[string]any{"id": importedID}}}}) + p.Reply("GET /api/experiments/templates/"+templateID, platformtest.Reply{Body: `{"id":"` + templateID + `","templateTitle":"Pod Crash","templateDescription":"d","placeholders":[],"tags":[],"lanes":[{"steps":[{"type":"wait","ignoreFailure":false,"parameters":{"duration":"5s"}}]}],"properties":{},"propertiesMetadata":[],"version":0,"created":"c","createdBy":{}}`}) + p.Reply("GET /api/integrations/webhook", platformtest.Reply{JSON: map[string]any{"content": []any{map[string]any{"id": webhookID}}}}) + p.Reply("GET /api/integrations/webhook/"+webhookID, platformtest.Reply{Body: storedWebhook}) + p.Reply("GET /api/integrations/slack", platformtest.Reply{JSON: map[string]any{"content": []any{map[string]any{"id": slackID}}}}) + p.Reply("GET /api/integrations/slack/"+slackID, platformtest.Reply{Body: storedSlack}) + p.Reply("GET /api/integrations/preflight", platformtest.Reply{JSON: map[string]any{"content": []any{}}}) + p.Reply("GET /api/integrations/preflight-action", platformtest.Reply{JSON: map[string]any{"content": []any{}}}) + p.Reply("GET /api/services/profiles", platformtest.Reply{JSON: map[string]any{"items": []any{ + map[string]any{"id": profileID, "name": "Shop", "origin": "CUSTOM"}, + map[string]any{"id": serviceID, "name": "Kubernetes Deployment", "origin": "STEADYBIT"}, + }}}) + p.Reply("GET /api/services/profiles/"+profileID, platformtest.Reply{Body: `{"id":"` + profileID + `","name":"Shop","origin":"CUSTOM","templates":[],"defaultProfile":false,"version":1}`}) + return p +} + +func TestExportTenantLeavesOutWhatThePlatformProvides(t *testing.T) { + p := fakeTenant(t) + dir := t.TempDir() + + out, err := platformtest.Stdout(t, func() error { return gitops.Export(ctx, p.Client, gitops.ExportOptions{Directory: dir, Tenant: true}) }) + + require.NoError(t, err) + assert.Equal(t, "Exported the tenant to "+dir+": 1 experiment templates, 1 environments, 1 teams, 1 property definitions, 1 hubs, 2 integrations, 1 service profiles.\n", out) + for file, content := range map[string]string{ + "property-definitions/tribe.yaml": "key: tribe\nlabel: Tribe\ndataType: STRING\n", + "environments/prod.yaml": "id: " + environmentID + "\nname: Prod\npredicate:\n operator: AND\n predicates: []\n", + "teams/adm.yaml": "key: ADM\nname: Admins\nallowedActions:\n - wait\n - service-validation\nallowedEnvironments:\n - Prod\nmanagedBy: MANUAL\n" + + "members:\n - username: u1\n email: jane@example.com\n role: OWNER\n", + "hubs/own-hub.yaml": "hubName: Own Hub\nrepositoryUrl: https://example.com/index.json\nid: " + hubID + "\n", + "templates/pod-crash.yaml": "id: " + templateID + "\ntemplateTitle: Pod Crash\ntemplateDescription: d\nplaceholders: []\ntags: []\nlanes:\n - steps:\n - type: wait\n ignoreFailure: false\n" + + " parameters:\n duration: 5s\nproperties: {}\npropertiesMetadata: []\n", + // Credentials are masked: the platform's mask of the secret gives away its length. + "integrations/webhook/notify.yaml": "id: " + webhookID + "\nscope: GLOBAL\nname: Notify\nurl: https://example.com/hook\nsecret: '********'\nevents:\n - '*'\n" + + "targetAttributeIncludes:\n - '*'\nheaders:\n Authorization: '********'\n", + "integrations/slack/chat.yaml": "id: " + slackID + "\nscope: GLOBAL\nname: Chat\nurl: '********'\nevents:\n - '*'\nchannel: '#chaos'\n", + "service-profiles/shop.yaml": "id: " + profileID + "\nname: Shop\norigin: CUSTOM\ntemplates: []\n", + } { + written, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(file))) + require.NoError(t, err, file) + assert.Equal(t, content, string(written), file) + } + for _, gone := range []string{"hubs/steadybit-reliability-hub.yaml", "templates/imported.yaml", "service-profiles/kubernetes-deployment.yaml", "integrations/preflight"} { + assert.NoFileExists(t, filepath.Join(dir, filepath.FromSlash(gone))) + } + assert.Empty(t, p.Requests("GET /api/experiments/templates/"+importedID), "a template imported from a hub is not even fetched") + listed := p.Requests("GET /api/experiments/templates")[0].Query + assert.Equal(t, []string{"true"}, listed["includeHidden"], "hidden templates are exported too") + assert.Equal(t, []string{"true"}, listed["includeNonAvailable"], "so are those whose actions are not available right now") + + out, err = platformtest.Stdout(t, func() error { return gitops.DiffProject(ctx, p.Client, dir) }) + require.NoError(t, err) + assert.Equal(t, 8, strings.Count(out, "match the platform"), out) +} + +// Applying an exported tenant leaves out the integrations that match: the masked secret +// in them could not be sent back, and leaving it out would remove it. +func TestApplyingAnExportedTenantSkipsMatchingIntegrations(t *testing.T) { + p := fakeTenant(t) + dir := t.TempDir() + _, err := platformtest.Stdout(t, func() error { return gitops.Export(ctx, p.Client, gitops.ExportOptions{Directory: dir, Tenant: true}) }) + require.NoError(t, err) + for _, route := range []string{"POST /api/properties/definitions", "POST /api/environments", "POST /api/teams", "POST /api/hubs", "POST /api/experiments/templates", "POST /api/services/profiles"} { + p.Reply(route, platformtest.Reply{JSON: map[string]any{}}) + } + + out, err := platformtest.Stdout(t, func() error { return gitops.ApplyProject(ctx, p.Client, gitops.ApplyOptions{Directory: dir}) }) + + require.NoError(t, err) + assert.Contains(t, out, "notify.yaml matches webhook integration "+webhookID+".\n") + assert.Contains(t, out, "chat.yaml matches Slack integration "+slackID+".\n") + assert.Empty(t, p.Requests("POST /api/integrations/webhook")) + assert.Empty(t, p.Requests("POST /api/integrations/slack")) + assert.Equal(t, []string{"true"}, p.Requests("POST /api/hubs")[0].Query["synchronize"], "the hub's templates are there for the service profiles") + order := []string{"Property definition tribe", "Environment", "Team ADM", "Hub", "Experiment template", "Service profile"} + last := -1 + for _, line := range order { + i := strings.Index(out, line) + assert.Greater(t, i, last, "%s is applied after what it depends on:\n%s", line, out) + last = i + } +} + +// Service profiles may name the templates a hub brings, so a hub that cannot be +// synchronized ends the apply before them. +func TestAHubThatCannotBeSynchronizedEndsTheApply(t *testing.T) { + p := fakeTenant(t) + dir := t.TempDir() + _, err := platformtest.Stdout(t, func() error { return gitops.Export(ctx, p.Client, gitops.ExportOptions{Directory: dir, Tenant: true}) }) + require.NoError(t, err) + for _, route := range []string{"POST /api/properties/definitions", "POST /api/environments", "POST /api/teams", "POST /api/experiments/templates", "POST /api/services/profiles"} { + p.Reply(route, platformtest.Reply{JSON: map[string]any{}}) + } + p.Reply("POST /api/hubs", platformtest.Reply{JSON: map[string]any{"id": hubID, "hubName": "Own Hub", "syncError": "index.json not found"}}) + + _, err = platformtest.Stdout(t, func() error { return gitops.ApplyProject(ctx, p.Client, gitops.ApplyOptions{Directory: dir}) }) + + assert.EqualError(t, err, "Hub Own Hub could not be synchronized: index.json not found") + assert.Empty(t, p.Requests("POST /api/experiments/templates")) + assert.Empty(t, p.Requests("POST /api/services/profiles")) +} + +// A changed integration is applied with the credentials the platform reads back in the +// clear; its secret it never does, so that has to be put in. +func TestAChangedExportedIntegrationKeepsItsCredentials(t *testing.T) { + p := fakeTenant(t) + dir := t.TempDir() + _, err := platformtest.Stdout(t, func() error { return gitops.Export(ctx, p.Client, gitops.ExportOptions{Directory: dir, Tenant: true}) }) + require.NoError(t, err) + for _, other := range []string{"property-definitions", "environments", "teams", "hubs", "templates", "service-profiles", "integrations/webhook"} { + require.NoError(t, os.RemoveAll(filepath.Join(dir, filepath.FromSlash(other)))) + } + slack := filepath.Join(dir, "integrations", "slack", "chat.yaml") + require.NoError(t, os.WriteFile(slack, []byte("id: "+slackID+"\nscope: GLOBAL\nname: Chat\nurl: '********'\nevents:\n - '*'\nchannel: '#incidents'\n"), 0o644)) + p.Reply("POST /api/integrations/slack", platformtest.Reply{JSON: map[string]any{"id": slackID, "name": "Chat"}}) + + out, err := platformtest.Stdout(t, func() error { + return gitops.ApplyProject(ctx, p.Client, gitops.ApplyOptions{Directory: dir}) + }) + + require.NoError(t, err) + assert.Equal(t, "Slack integration Chat ("+slackID+") updated.\n", out) + sent := p.Requests("POST /api/integrations/slack")[0].JSON(t).(map[string]any) + assert.Equal(t, "https://hooks.example.com/T1/B2/xyz", sent["url"]) + assert.Equal(t, "#incidents", sent["channel"]) + written, _ := os.ReadFile(slack) + assert.Contains(t, string(written), "url: '********'\n", "the credential is not written to the file") +} + +func TestCredentialsAreNeverPrinted(t *testing.T) { + p := platformtest.New(t) + p.Reply("GET /api/integrations/webhook/"+webhookID, platformtest.Reply{Body: storedWebhook}) + file := write(t, "w.yml", "id: "+webhookID+"\nscope: GLOBAL\nname: Notify\nurl: https://example.com/hook\nsecret: n3w-s3cret\nevents:\n - '*'\n"+ + "targetAttributeIncludes:\n - '*'\nheaders:\n Authorization: Bearer other\n") + + out, err := platformtest.Stdout(t, func() error { + return gitops.DiffFiles(ctx, p.Client, gitops.Integrations["webhook"], []string{file}, false) + }) + + assert.ErrorIs(t, err, gitops.ErrDifferent) + assert.Contains(t, out, "-secret: '********'\n+secret: '******** (from the file)'\n") + assert.Contains(t, out, "- Authorization: '********'\n+ Authorization: '******** (from the file)'\n") + assert.NotContains(t, out, "s3cret") + assert.NotContains(t, out, "Bearer") + + // An unquoted number in the file is a credential all the same. + file = write(t, "n.yml", "id: "+webhookID+"\nscope: GLOBAL\nname: Notify\nurl: https://example.com/hook\nsecret: '********'\nevents:\n - '*'\n"+ + "targetAttributeIncludes:\n - '*'\nheaders:\n Authorization: Bearer abc\n X-Api-Key: 8675309\n") + + out, err = platformtest.Stdout(t, func() error { + return gitops.DiffFiles(ctx, p.Client, gitops.Integrations["webhook"], []string{file}, false) + }) + + assert.ErrorIs(t, err, gitops.ErrDifferent) + assert.Contains(t, out, "+ X-Api-Key: '******** (from the file)'\n") + assert.NotContains(t, out, "8675309") +} + +// A team sent no actions may still wait and validate services, and a webhook sent no +// target attributes reports all of them. Neither is a difference. +func TestAHandWrittenTeamAndWebhookMatchRightAfterApply(t *testing.T) { + p := platformtest.New(t) + p.Reply("GET /api/teams/CLIX", platformtest.Reply{Body: `{"id":"` + teamID + `","key":"CLIX","name":"x","allowedActions":["wait","service-validation"],"allowedEnvironments":["Prod"],"managedBy":"MANUAL","version":0,"members":[]}`}) + p.Reply("GET /api/integrations/webhook/"+webhookID, platformtest.Reply{Body: `{"id":"` + webhookID + `","version":0,"scope":"GLOBAL","name":"n","url":"https://example.com","events":["*"],"targetAttributeIncludes":["*"],"headers":{}}`}) + team := write(t, "team.yml", "key: CLIX\nname: x\nallowedActions: []\nallowedEnvironments:\n - Prod\n") + webhook := write(t, "webhook.yml", "id: "+webhookID+"\nscope: GLOBAL\nname: n\nurl: https://example.com\nevents:\n - '*'\ntargetAttributeIncludes: []\n") + + _, err := platformtest.Stdout(t, func() error { return gitops.DiffFiles(ctx, p.Client, gitops.Team, []string{team}, false) }) + require.NoError(t, err) + _, err = platformtest.Stdout(t, func() error { + return gitops.DiffFiles(ctx, p.Client, gitops.Integrations["webhook"], []string{webhook}, false) + }) + require.NoError(t, err) + + p.Reply("GET /api/teams/CLIX", platformtest.Reply{Body: `{"key":"CLIX","name":"x","allowedActions":["wait"],"allowedEnvironments":["Prod"]}`}) + out, err := platformtest.Stdout(t, func() error { return gitops.DiffFiles(ctx, p.Client, gitops.Team, []string{team}, false) }) + assert.ErrorIs(t, err, gitops.ErrDifferent) + assert.Contains(t, out, "-allowedActions:\n- - wait\n+allowedActions: []\n") +} + +// On another platform the team has another id. It is the same team by its key, and +// applying the file sends no id to contradict it. +func TestATeamMatchesByKeyOnAnotherPlatform(t *testing.T) { + p := platformtest.New(t) + p.Reply("GET /api/teams/ADM", platformtest.Reply{Body: `{"id":"` + environmentID + `","key":"ADM","name":"Admins","allowedActions":["wait","service-validation"],"allowedEnvironments":["Prod"],"managedBy":"MANUAL","version":0,"members":[]}`}) + p.Reply("POST /api/teams", platformtest.Reply{JSON: map[string]any{}}) + dir := t.TempDir() + file := filepath.Join(dir, "teams", "adm.yaml") + require.NoError(t, os.MkdirAll(filepath.Dir(file), 0o755)) + require.NoError(t, os.WriteFile(file, []byte("id: "+teamID+"\nkey: ADM\nname: Admins\nallowedActions:\n - wait\n - service-validation\nallowedEnvironments:\n - Prod\nmembers: []\n"), 0o644)) + + out, err := platformtest.Stdout(t, func() error { return gitops.DryRun(ctx, p.Client, gitops.Team, []string{file}, false) }) + require.NoError(t, err) + assert.Equal(t, file+" matches team ADM.\n", out) + + _, err = platformtest.Stdout(t, func() error { return gitops.ApplyProject(ctx, p.Client, gitops.ApplyOptions{Directory: dir}) }) + require.NoError(t, err) + assert.NotContains(t, p.Requests("POST /api/teams")[0].JSON(t), "id") +} + +func TestEnvironmentsAreFoundByName(t *testing.T) { + p := platformtest.New(t) + p.Reply("GET /api/environments", platformtest.Reply{JSON: map[string]any{"environments": []any{ + map[string]any{"id": teamID, "name": "Prod EU"}, + map[string]any{"id": environmentID, "name": "Prod"}, + }}}) + p.Reply("GET /api/environments/"+environmentID, platformtest.Reply{Body: `{"id":"` + environmentID + `","name":"Prod","version":2,"predicate":{"operator":"AND","predicates":[]},"state":"READY"}`}) + file := write(t, "e.yml", "name: Prod\npredicate:\n operator: AND\n predicates: []\n") + + out, err := platformtest.Stdout(t, func() error { return gitops.DryRun(ctx, p.Client, gitops.Environment, []string{file}, false) }) + + require.NoError(t, err) + assert.Equal(t, file+" matches environment "+environmentID+".\n", out) + assert.Equal(t, []string{"Prod"}, p.Requests("GET /api/environments")[0].Query["search"]) +} + +func TestAnUnknownTeamWouldBeCreated(t *testing.T) { + p := platformtest.New(t) + p.Reply("GET /api/teams/NEW", platformtest.Reply{Status: http.StatusNotFound}) + file := write(t, "t.yml", "key: NEW\nname: New\nallowedActions: []\nallowedEnvironments: []\n") + + out, err := platformtest.Stdout(t, func() error { return gitops.DryRun(ctx, p.Client, gitops.Team, []string{file}, false) }) + + require.NoError(t, err) + assert.Equal(t, file+" would create a new team.\n", out) +} + +// An id that is no UUID is reported, not sent to the platform as the zero UUID. +func TestAnInvalidIDIsReported(t *testing.T) { + p := fakeTenant(t) + p.Reply("GET /api/hubs", platformtest.Reply{JSON: map[string]any{"hubs": []any{map[string]any{"id": "not-an-id"}}}}) + + _, err := platformtest.Stdout(t, func() error { + return gitops.Export(ctx, p.Client, gitops.ExportOptions{Directory: t.TempDir(), Tenant: true}) + }) + + assert.EqualError(t, err, "Failed to get hub not-an-id: not a valid id") + + file := write(t, "hub.yml", "hubName: Own Hub\nrepositoryUrl: https://example.com/index.json\nid: not-an-id\n") + _, err = platformtest.Stdout(t, func() error { return gitops.DiffFiles(ctx, p.Client, gitops.Hub, []string{file}, false) }) + assert.EqualError(t, err, "Failed to get the hub for "+file+": not-an-id is not a valid id") +} diff --git a/internal/hub/hub.go b/internal/hub/hub.go index 730acd5..f9f59e0 100644 --- a/internal/hub/hub.go +++ b/internal/hub/hub.go @@ -20,7 +20,7 @@ import ( // What the last synchronisation found and who edited the hub is the platform's. The // version is dropped as `service get` drops it. -var readOnly = []string{"version", "templates", "lastSync", "lastRepositoryChange", "syncError", "created", "createdBy", "edited", "editedBy"} +var ReadOnly = []string{"version", "templates", "lastSync", "lastRepositoryChange", "syncError", "created", "createdBy", "edited", "editedBy"} func uuid(id string) (openapi_types.UUID, error) { u, ok := resource.UUID(id) @@ -77,7 +77,7 @@ func Get(ctx context.Context, c *platform.Client, o GetOptions) error { if err != nil { return notFoundOr(err, o.ID, "Failed to get hub %s") } - if err := resource.Output(resource.Strip(doc, readOnly...), o.File, o.Type); err != nil { + if err := resource.Output(resource.Strip(doc, ReadOnly...), o.File, o.Type); err != nil { return err } if o.File != "" { @@ -107,7 +107,7 @@ func Apply(ctx context.Context, c *platform.Client, o ApplyOptions) error { Templates []any `json:"templates"` SyncError string `json:"syncError"` } - resp, err := c.UpsertHubWithBody(ctx, &api.UpsertHubParams{Synchronize: &o.Synchronize}, "application/json", resource.Body(resource.Strip(doc, readOnly...).Value())) + resp, err := c.UpsertHubWithBody(ctx, &api.UpsertHubParams{Synchronize: &o.Synchronize}, "application/json", resource.Body(resource.Strip(doc, ReadOnly...).Value())) resp, err = platform.Decode(resp, err, &saved) if err != nil { return resource.Applied{}, platform.Failed(err, "Failed to save hub %s", name) diff --git a/internal/integration/integration.go b/internal/integration/integration.go index 0dc0067..1ad1c7c 100644 --- a/internal/integration/integration.go +++ b/internal/integration/integration.go @@ -13,6 +13,7 @@ import ( "strings" openapi_types "github.com/oapi-codegen/runtime/types" + "github.com/steadybit/cli/v6/internal/jsyaml" "github.com/steadybit/cli/v6/internal/output" "github.com/steadybit/cli/v6/internal/platform" "github.com/steadybit/cli/v6/internal/resource" @@ -21,7 +22,7 @@ import ( // The version is dropped as `service get` drops it: kept in a file, it turns every apply // after an edit in the UI into a conflict. -var readOnly = []string{"version"} +var ReadOnly = []string{"version"} type Kind struct { Name string // as the command names it @@ -29,6 +30,9 @@ type Kind struct { Plural string // The column that says where the integration reports to. Column, ColumnTitle string + // Secrets are the fields holding credentials; each value of a map among them is one. + // `export` masks them, so that a tenant kept in Git does not hold them. + Secrets []string list func(ctx context.Context, c *platform.Client) (*http.Response, error) get func(ctx context.Context, c *platform.Client, id openapi_types.UUID) (*http.Response, error) @@ -39,7 +43,9 @@ type Kind struct { var ( Webhook = Kind{ Name: "webhook", Title: "Webhook integration", Plural: "webhook integrations", Column: "url", ColumnTitle: "URL", - list: func(ctx context.Context, c *platform.Client) (*http.Response, error) { return c.GetCustomWebhooks(ctx) }, + // Headers carry API keys and tokens as often as anything else. + Secrets: []string{"secret", "headers"}, + list: func(ctx context.Context, c *platform.Client) (*http.Response, error) { return c.GetCustomWebhooks(ctx) }, get: func(ctx context.Context, c *platform.Client, id openapi_types.UUID) (*http.Response, error) { return c.GetCustomWebhook(ctx, id) }, @@ -52,6 +58,8 @@ var ( } Slack = Kind{ Name: "slack", Title: "Slack integration", Plural: "Slack integrations", Column: "channel", ColumnTitle: "Channel", + // The URL of a Slack incoming webhook is its credential: whoever has it can post. + Secrets: []string{"url"}, list: func(ctx context.Context, c *platform.Client) (*http.Response, error) { return c.GetSlackIntegrations(ctx) }, @@ -67,6 +75,7 @@ var ( } Preflight = Kind{ Name: "preflight", Title: "Preflight webhook", Plural: "preflight webhooks", Column: "url", ColumnTitle: "URL", + Secrets: []string{"secret", "headers"}, list: func(ctx context.Context, c *platform.Client) (*http.Response, error) { return c.GetPreflightWebhooks(ctx) }, @@ -98,6 +107,140 @@ var ( Kinds = []Kind{Webhook, Slack, Preflight, PreflightAction} ) +// Fetch gets one integration as `get` reads it. +func (k Kind) Fetch(ctx context.Context, c *platform.Client, id openapi_types.UUID) (*http.Response, error) { + return k.get(ctx, c, id) +} + +// FetchAll lists the integrations of the kind, as `list` reads them. +func (k Kind) FetchAll(ctx context.Context, c *platform.Client) (*http.Response, error) { + return k.list(ctx, c) +} + +// Mask is what `export` writes in place of a credential. Like the platform's own mask of +// a secret, it is only asterisks, which is how apply and diff tell one. +const Mask = "********" + +func Masked(v any) bool { + s, ok := v.(string) + return ok && s != "" && strings.Trim(s, "*") == "" +} + +// MaskSecrets replaces the credentials in an integration with the mask. +func (k Kind) MaskSecrets(doc *jsyaml.Map) { + for _, field := range k.Secrets { + switch v, _ := doc.Get(field); x := v.(type) { + case string: + if x != "" { + doc.Set(field, Mask) + } + case *jsyaml.Map: + for _, key := range x.Keys() { + if s, _ := x.Get(key); s != "" { + x.Set(key, Mask) + } + } + } + } +} + +// stored reads, once and only when asked, what the platform holds for the integration +// the file names: nil when it names none. +func (k Kind) stored(ctx context.Context, c *platform.Client, doc *jsyaml.Map) func() (*jsyaml.Map, error) { + var stored *jsyaml.Map + loaded := false + return func() (*jsyaml.Map, error) { + if !loaded { + loaded = true + id, _ := doc.Get("id") + if u, ok := resource.UUID(fmt.Sprint(id)); ok { + d, _, err := platform.ReadDocument(k.get(ctx, c, u)) + if err != nil && !platform.IsStatus(err, http.StatusNotFound) { + return nil, platform.Failed(err, "Failed to get %s %s", k.Title, id) + } + if d != nil { + stored = d.Value() + } + } + } + return stored, nil + } +} + +// unchanged tells a file that holds what the platform does, its secret aside, which a +// mask in the file stands for. +func unchanged(doc, stored *jsyaml.Map) bool { + if stored == nil { + return false + } + if secret, _ := stored.Get("secret"); secret == nil || secret == "" { + return false + } + without := func(m *jsyaml.Map) string { + c := jsyaml.Clone(m).(*jsyaml.Map) + for _, f := range append([]string{"secret"}, ReadOnly...) { + c.Delete(f) + } + return jsyaml.CompactJSON(c) + } + return without(doc) == without(stored) +} + +// keepStored puts back what the platform holds in place of the masks `export` writes +// for the credentials it reads back in the clear, so that an exported file applies as +// it is. The secret it never reads back; a masked one is refused afterwards. +func (k Kind) keepStored(file string, doc *jsyaml.Map, load func() (*jsyaml.Map, error)) error { + value := func(field, key string) (any, error) { + stored, err := load() + if err != nil { + return nil, err + } + name := field + var v any + if stored != nil { + v, _ = stored.Get(field) + if key != "" { + name = field + "." + key + m, _ := v.(*jsyaml.Map) + v = nil + if m != nil { + v, _ = m.Get(key) + } + } + } + if v == nil || v == "" || Masked(v) { + return nil, fmt.Errorf("%s file '%s' holds a masked %s, and the platform has none to keep. Put the value in.", k.Title, file, name) + } + return v, nil + } + for _, field := range k.Secrets { + if field == "secret" { + continue + } + switch v, _ := doc.Get(field); x := v.(type) { + case string: + if Masked(x) { + kept, err := value(field, "") + if err != nil { + return err + } + doc.Set(field, kept) + } + case *jsyaml.Map: + for _, key := range x.Keys() { + if s, _ := x.Get(key); Masked(s) { + kept, err := value(field, key) + if err != nil { + return err + } + x.Set(key, kept) + } + } + } + } + return nil +} + func (k Kind) uuid(id string) (openapi_types.UUID, error) { u, ok := resource.UUID(id) if !ok { @@ -157,7 +300,7 @@ func Get(ctx context.Context, c *platform.Client, k Kind, o GetOptions) error { if err != nil { return k.notFoundOr(err, o.ID, "Failed to get %s %s") } - if err := resource.Output(resource.Strip(doc, readOnly...), o.File, o.Type); err != nil { + if err := resource.Output(resource.Strip(doc, ReadOnly...), o.File, o.Type); err != nil { return err } if o.File != "" { @@ -177,13 +320,27 @@ func Apply(ctx context.Context, c *platform.Client, k Kind, o ApplyOptions) erro if name == "" { return resource.Applied{}, fmt.Errorf("%s file '%s' does not name the integration.", k.Title, file) } + stored := k.stored(ctx, c, doc.Value()) + if err := k.keepStored(file, doc.Value(), stored); err != nil { + return resource.Applied{}, err + } // The platform masks secrets when reading them back and rejects the mask, while - // leaving the secret out removes it. Neither is what a file from `get` means. - if secret, _ := doc.Get("secret"); secret != "" && strings.Trim(secret, "*") == "" { - return resource.Applied{}, fmt.Errorf("%s file '%s' holds the masked secret `get` writes. Put the secret in, or remove it for none.", k.Title, file) + // leaving the secret out removes it. Neither is what a file from `get` means. Left + // as it was written, though, the file has nothing to apply, as `apply -d` finds. + if secret, _ := doc.Value().Get("secret"); Masked(secret) { + held, err := stored() + if err != nil { + return resource.Applied{}, err + } + if unchanged(doc.Value(), held) { + id, _ := doc.Get("id") + fmt.Printf("%s %s (%s) unchanged.\n", k.Title, name, id) + return resource.Applied{}, nil + } + return resource.Applied{}, fmt.Errorf("%s file '%s' holds the masked secret `get` and `export` write. Put the secret in, or remove it for none.", k.Title, file) } var saved struct{ ID, Name string } - resp, err := k.upsert(ctx, c, resource.Body(resource.Strip(doc, readOnly...).Value())) + resp, err := k.upsert(ctx, c, resource.Body(resource.Strip(doc, ReadOnly...).Value())) resp, err = platform.Decode(resp, err, &saved) if err != nil { return resource.Applied{}, platform.Failed(err, "Failed to save %s %s", k.Title, name) diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go index 08a7c02..282d2c5 100644 --- a/internal/integration/integration_test.go +++ b/internal/integration/integration_test.go @@ -72,6 +72,7 @@ func TestGetAndApplyRoundTrip(t *testing.T) { func TestApplyCreatesAndRefusesAMaskedSecret(t *testing.T) { p := platformtest.New(t) p.Reply("POST /api/integrations/webhook", platformtest.Reply{Status: http.StatusCreated, JSON: map[string]any{"id": id, "name": "Notify"}}) + p.Reply("GET /api/integrations/webhook/"+id, platformtest.Reply{Status: http.StatusNotFound}) dir := t.TempDir() file := filepath.Join(dir, "webhook.yml") require.NoError(t, os.WriteFile(file, []byte("name: Notify\nscope: GLOBAL\nurl: https://example.com\nsecret: s3cret\n"), 0o644)) @@ -87,10 +88,59 @@ func TestApplyCreatesAndRefusesAMaskedSecret(t *testing.T) { content, _ := os.ReadFile(file) assert.Equal(t, "id: "+id+"\nname: Notify\nscope: GLOBAL\nurl: https://example.com\nsecret: s3cret\n", string(content)) err = integration.Apply(ctx, p.Client, integration.Webhook, integration.ApplyOptions{Files: []string{masked}}) - assert.EqualError(t, err, "Webhook integration file '"+masked+"' holds the masked secret `get` writes. Put the secret in, or remove it for none.") + assert.EqualError(t, err, "Webhook integration file '"+masked+"' holds the masked secret `get` and `export` write. Put the secret in, or remove it for none.") assert.Len(t, p.Requests("POST /api/integrations/webhook"), 1) } +// A header masked by `export` is sent as the platform holds it; one the platform does not +// have cannot be. +func TestApplyKeepsTheStoredValueOfAMaskedHeader(t *testing.T) { + p := platformtest.New(t) + p.Reply("GET /api/integrations/webhook/"+id, platformtest.Reply{Body: `{"id":"` + id + `","name":"Notify","headers":{"Authorization":"Bearer abc"}}`}) + p.Reply("POST /api/integrations/webhook", platformtest.Reply{JSON: map[string]any{"id": id, "name": "Notify"}}) + dir := t.TempDir() + file := filepath.Join(dir, "webhook.yml") + require.NoError(t, os.WriteFile(file, []byte("id: "+id+"\nname: Notify\nheaders:\n Authorization: '********'\n X-Team: '********'\n"), 0o644)) + + err := integration.Apply(ctx, p.Client, integration.Webhook, integration.ApplyOptions{Files: []string{file}}) + assert.EqualError(t, err, "Webhook integration file '"+file+"' holds a masked headers.X-Team, and the platform has none to keep. Put the value in.") + + require.NoError(t, os.WriteFile(file, []byte("id: "+id+"\nname: Notify\nheaders:\n Authorization: '********'\n X-Team: chaos\n"), 0o644)) + _, err = platformtest.Stdout(t, func() error { + return integration.Apply(ctx, p.Client, integration.Webhook, integration.ApplyOptions{Files: []string{file}}) + }) + + require.NoError(t, err) + sent := p.Requests("POST /api/integrations/webhook") + require.Len(t, sent, 1) + assert.Equal(t, map[string]any{"Authorization": "Bearer abc", "X-Team": "chaos"}, sent[0].JSON(t).(map[string]any)["headers"]) + content, _ := os.ReadFile(file) + assert.Contains(t, string(content), "Authorization: '********'", "the stored value is not written to the file") +} + +// An exported file left as it is has nothing to apply, masked secret and all, as +// `apply -d` finds; changed, it needs the secret put in. +func TestApplySkipsAnUnchangedFileWithAMaskedSecret(t *testing.T) { + p := platformtest.New(t) + p.Reply("GET /api/integrations/webhook/"+id, platformtest.Reply{Body: `{"id":"` + id + `","version":2,"name":"Notify","url":"https://example.com","secret":"****************","headers":{"Authorization":"Bearer abc"}}`}) + file := filepath.Join(t.TempDir(), "webhook.yml") + require.NoError(t, os.WriteFile(file, []byte("id: "+id+"\nname: Notify\nurl: https://example.com\nsecret: '********'\nheaders:\n Authorization: '********'\n"), 0o644)) + + out, err := platformtest.Stdout(t, func() error { + return integration.Apply(ctx, p.Client, integration.Webhook, integration.ApplyOptions{Files: []string{file}}) + }) + + require.NoError(t, err) + assert.Equal(t, "Webhook integration Notify ("+id+") unchanged.\n", out) + assert.Empty(t, p.Requests("POST /api/integrations/webhook")) + assert.Len(t, p.Requests("GET /api/integrations/webhook/"+id), 1, "the platform's version is read once") + + require.NoError(t, os.WriteFile(file, []byte("id: "+id+"\nname: Notify\nurl: https://example.org\nsecret: '********'\nheaders:\n Authorization: '********'\n"), 0o644)) + err = integration.Apply(ctx, p.Client, integration.Webhook, integration.ApplyOptions{Files: []string{file}}) + assert.EqualError(t, err, "Webhook integration file '"+file+"' holds the masked secret `get` and `export` write. Put the secret in, or remove it for none.") + assert.Empty(t, p.Requests("POST /api/integrations/webhook")) +} + func TestDelete(t *testing.T) { p := platformtest.New(t) p.Reply("DELETE /api/integrations/slack/"+id, platformtest.Reply{JSON: map[string]any{"id": id}}) diff --git a/internal/property/property.go b/internal/property/property.go index b234d78..1091ffd 100644 --- a/internal/property/property.go +++ b/internal/property/property.go @@ -21,7 +21,7 @@ import ( // The version is dropped as `service get` drops it: kept in a file, it turns every apply // after an edit in the UI into a conflict. -var readOnly = []string{"version"} +var ReadOnly = []string{"version"} func definitionNotFoundOr(err error, key, format string) error { if platform.IsStatus(err, http.StatusNotFound) { @@ -84,7 +84,7 @@ func GetDefinition(ctx context.Context, c *platform.Client, o GetDefinitionOptio if err != nil { return definitionNotFoundOr(err, o.Key, "Failed to get property definition %s") } - if err := resource.Output(resource.Strip(doc, readOnly...), o.File, o.Type); err != nil { + if err := resource.Output(resource.Strip(doc, ReadOnly...), o.File, o.Type); err != nil { return err } if o.File != "" { @@ -107,7 +107,7 @@ func ApplyDefinitions(ctx context.Context, c *platform.Client, o ApplyDefinition return resource.Applied{}, fmt.Errorf("Property definition file '%s' does not name the key.", file) } resp, err := c.UpsertPropertyDefinitionWithBody(ctx, &api.UpsertPropertyDefinitionParams{DeleteValues: &o.DeleteValues}, "application/json", - resource.Body(resource.Strip(doc, readOnly...).Value())) + resource.Body(resource.Strip(doc, ReadOnly...).Value())) _, resp, err = platform.Read(resp, err) if err != nil { return resource.Applied{}, platform.Failed(err, "Failed to save property definition %s", key) @@ -245,7 +245,7 @@ func GetAssociation(ctx context.Context, c *platform.Client, o GetAssociationOpt if err != nil { return associationNotFoundOr(err, o.ID, "Failed to get property association %s") } - if err := resource.Output(resource.Strip(doc, readOnly...), o.File, o.Type); err != nil { + if err := resource.Output(resource.Strip(doc, ReadOnly...), o.File, o.Type); err != nil { return err } if o.File != "" { @@ -266,7 +266,7 @@ func ApplyAssociations(ctx context.Context, c *platform.Client, o ApplyAssociati return resource.Applied{}, fmt.Errorf("Property association file '%s' does not name the property key.", file) } var saved struct{ ID, Key string } - resp, err := c.UpsertPropertyAssociationWithBody(ctx, "application/json", resource.Body(resource.Strip(doc, readOnly...).Value())) + resp, err := c.UpsertPropertyAssociationWithBody(ctx, "application/json", resource.Body(resource.Strip(doc, ReadOnly...).Value())) resp, err = platform.Decode(resp, err, &saved) if err != nil { return resource.Applied{}, platform.Failed(err, "Failed to save the property association of %s", key) diff --git a/internal/team/team.go b/internal/team/team.go index 5cb55c0..86a293f 100644 --- a/internal/team/team.go +++ b/internal/team/team.go @@ -22,8 +22,8 @@ import ( // The version is dropped as `service get` drops it. Members are sent back as the platform // takes them, by username, email and role; the rest describes the user. var ( - readOnly = []string{"version"} - memberReadOnly = []string{"name", "pictureUrl", "managedBy"} + ReadOnly = []string{"version"} + MemberReadOnly = []string{"name", "pictureUrl", "managedBy"} ) func notFoundOr(err error, key, format string) error { @@ -85,12 +85,12 @@ func Get(ctx context.Context, c *platform.Client, o GetOptions) error { if err != nil { return notFoundOr(err, o.Key, "Failed to get team %s") } - resource.Strip(doc, readOnly...) + resource.Strip(doc, ReadOnly...) if members, ok := doc.Value().Get("members"); ok { list, _ := members.([]any) for _, m := range list { if member, ok := m.(*jsyaml.Map); ok { - for _, field := range memberReadOnly { + for _, field := range MemberReadOnly { member.Delete(field) } } @@ -111,13 +111,15 @@ type ApplyOptions struct { } // Apply upserts teams by their key, which is what names a team; there is no id to write back. +// The id `get` writes is not sent: the platform goes by the key alone, and another +// platform's id in the file would only contradict it. func Apply(ctx context.Context, c *platform.Client, o ApplyOptions) error { return resource.ApplyFiles(o.Files, o.Recursive, "team", func(file string, doc *output.Document) (resource.Applied, error) { key, _ := doc.Get("key") if key == "" { return resource.Applied{}, fmt.Errorf("Team file '%s' does not name the team key.", file) } - resp, err := c.UpsertTeamWithBody(ctx, &api.UpsertTeamParams{}, "application/json", resource.Body(resource.Strip(doc, readOnly...).Value())) + resp, err := c.UpsertTeamWithBody(ctx, &api.UpsertTeamParams{}, "application/json", resource.Body(resource.Strip(doc, append([]string{"id"}, ReadOnly...)...).Value())) _, resp, err = platform.Read(resp, err) if err != nil { return resource.Applied{}, platform.Failed(err, "Failed to save team %s", key) diff --git a/internal/team/team_test.go b/internal/team/team_test.go index 78d6dac..cfe3016 100644 --- a/internal/team/team_test.go +++ b/internal/team/team_test.go @@ -55,6 +55,8 @@ func TestGetAndApplyRoundTripByKey(t *testing.T) { sent := p.Requests("POST /api/teams")[0].JSON(t).(map[string]any) assert.Equal(t, []any{map[string]any{"username": "u-1", "email": "jane@example.com", "role": "OWNER"}}, sent["members"]) assert.Equal(t, "MANUAL", sent["managedBy"]) + assert.Contains(t, string(before), "id: 0190d7b2-0000-7000-8000-000000000001") + assert.NotContains(t, sent, "id", "the platform goes by the key") } func TestApplyWritesNothingBack(t *testing.T) { diff --git a/internal/template/template.go b/internal/template/template.go index 1a0ea10..6f0314c 100644 --- a/internal/template/template.go +++ b/internal/template/template.go @@ -23,7 +23,7 @@ import ( // Who created and edited a template cannot be sent back. The version is dropped as // `service get` drops it, so that an edit in the UI does not turn the next apply into a // conflict. -var readOnly = []string{"created", "createdBy", "edited", "editedBy", "version"} +var ReadOnly = []string{"created", "createdBy", "edited", "editedBy", "version"} func notFoundOr(err error, id, format string) error { if platform.IsStatus(err, http.StatusNotFound) { @@ -118,7 +118,7 @@ func Get(ctx context.Context, c *platform.Client, o GetOptions) error { } doc = output.NewDocument(values) } - if err := resource.Output(resource.Strip(doc, readOnly...), o.File, o.Type); err != nil { + if err := resource.Output(resource.Strip(doc, ReadOnly...), o.File, o.Type); err != nil { return err } if o.File != "" { @@ -139,7 +139,7 @@ func Apply(ctx context.Context, c *platform.Client, o ApplyOptions) error { return resource.Applied{}, fmt.Errorf("Template file '%s' does not name a templateTitle.", file) } var saved struct{ ID, TemplateTitle string } - resp, err := c.UpsertExperimentTemplateWithBody(ctx, "application/json", resource.Body(resource.Strip(doc, readOnly...).Value())) + resp, err := c.UpsertExperimentTemplateWithBody(ctx, "application/json", resource.Body(resource.Strip(doc, ReadOnly...).Value())) resp, err = platform.Decode(resp, err, &saved) if err != nil { return resource.Applied{}, platform.Failed(err, "Failed to save experiment template %s", title)