From d99b2dcfd9faed6c101130e75cabcafcd74c0d21 Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Tue, 29 Sep 2026 12:04:53 +0200 Subject: [PATCH 1/8] feat: keep the tenant's configuration in Git with export --tenant Admins want templates, environments, teams, property definitions, hubs and integrations under version control the way a team keeps its experiments. `export --tenant -d dir` writes them one directory per kind, and `diff -d` and `apply -d [--dry-run]` recognise those directories next to a team's. Apply follows the references between kinds: property definitions, environments (teams name them), teams (integrations and experiments name them), hubs, templates (service profiles name them), integrations, then service profiles, services, experiments and schedules. What every platform provides is left out: the two hubs the platform connects itself (fixed ids), templates imported from a hub (they keep the hub's id, and `template import` brings them back) and Steadybit's service profiles. The platform masks webhook secrets on read, refuses the mask on write and drops the secret when it is left out, so neither keeping nor removing it in the file round-trips. Exported files hold a mask for every credential, including header values and Slack webhook URLs, which the platform returns in the clear. A mask matches whatever the platform holds, so an export shows no drift; the project apply skips integrations that match, and `integration apply` sends the stored header values and URLs in place of masks. A secret has to be put in to change its integration. Diffs never print credentials. The platform turns a team's empty allowedActions into [wait, service-validation] and a webhook's empty targetAttributeIncludes into ['*']; Kind.Defaults now also covers a field the file leaves empty, so a hand-written file is not drift right after it is applied. Team members' read-only fields are stripped by path, as `team get` does. Each new kind also gets its own `diff` and `apply --dry-run`. --- CHANGELOG.md | 19 ++ README.md | 28 ++- internal/cli/environment.go | 4 +- internal/cli/experiment.go | 2 +- internal/cli/gitops.go | 16 +- internal/cli/hub.go | 4 +- internal/cli/integration.go | 4 +- internal/cli/property.go | 4 +- internal/cli/schedule.go | 2 +- internal/cli/service.go | 4 +- internal/cli/team.go | 4 +- internal/cli/template.go | 4 +- internal/environment/environment.go | 6 +- internal/gitops/kinds.go | 236 ++++++++++++++++++++++- internal/gitops/project.go | 107 +++++++--- internal/gitops/project_test.go | 2 +- internal/gitops/tenant.go | 221 +++++++++++++++++++++ internal/gitops/tenant_test.go | 217 +++++++++++++++++++++ internal/hub/hub.go | 6 +- internal/integration/integration.go | 127 +++++++++++- internal/integration/integration_test.go | 28 ++- internal/property/property.go | 10 +- internal/team/team.go | 10 +- internal/template/template.go | 6 +- 24 files changed, 993 insertions(+), 78 deletions(-) create mode 100644 internal/gitops/tenant.go create mode 100644 internal/gitops/tenant_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index a2723f4..2ac3ac4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## v6.1.0 (unreleased) + +- `export --tenant -d dir` writes the tenant's configuration, to keep in Git: experiment + templates, environments, teams, property definitions, hubs, webhook, Slack and preflight + integrations and custom service profiles. `diff -d dir` and `apply -d dir [--dry-run]` + take such a directory as they take a team's, and apply property definitions, + environments, teams, hubs, templates, integrations and profiles in that order, before + services, experiments and schedules. Hubs the platform connects itself, templates + imported from a hub and Steadybit's service profiles are left out, as every platform + has them. +- Credentials of integrations are masked in exported files, and never printed by `diff`. + A masked value matches whatever the platform holds, so that an exported tenant shows no + drift and applies again: integrations that match are not applied, as the platform keeps + no secret it is not sent, and `integration ... apply` sends the stored header values and + Slack URLs in place of their masks. +- Experiment templates, environments, teams, property definitions, hubs and integrations + have a `diff`, and their `apply` a `--dry-run`. The actions a team is given when sent + none, and the target attributes a webhook reports when sent none, are not differences. + ## v6.0.1 - `experiment apply` and `experiment run` no longer send a `version` from the file. The diff --git a/README.md b/README.md index 91871d1..eaab9e1 100644 --- a/README.md +++ b/README.md @@ -265,9 +265,33 @@ steadybit apply -d ./chaos --dry-run # what an apply would create or update steadybit apply -d ./chaos # profiles, services, experiments, then schedules ``` +Keep the tenant's configuration in Git the same way: experiment templates, environments, +teams, property definitions, hubs, integrations and custom service profiles, one directory +each (`templates/`, `environments/`, `teams/`, `property-definitions/`, `hubs/`, +`integrations//`, `service-profiles/`): + +```bash +steadybit export --tenant -d ./platform # needs an admin access token +steadybit diff -d ./platform +steadybit apply -d ./platform --dry-run +steadybit apply -d ./platform # definitions, environments, teams, hubs, templates, integrations, profiles +``` + +What the platform provides is left out: the hubs it connects, the templates imported from +a hub (`template import` brings them back) and Steadybit's service profiles. + +Credentials of integrations (secrets, header values, Slack webhook URLs) are written as +`'********'`. A mask stands for what the platform holds: `diff` does not report it, and +`apply` leaves out the integrations that match the platform and sends the stored header +values and URLs in place of their masks. The platform never reads a secret back, so to +change an integration that has one, put the secret in, e.g. from a CI secret, before +applying. As the platform cannot say whether that is the secret it holds, such a file is +always a difference. `diff` never prints credentials. + Each kind also has its own `diff`, and its `apply` a `--dry-run`, e.g. -`steadybit experiment diff -f ./experiments -R`. Fields the platform fills in with defaults -are not reported as differences. +`steadybit experiment diff -f ./experiments -R` or +`steadybit integration webhook diff -f ./platform/integrations/webhook -R`. Fields the +platform fills in with defaults are not reported as differences. ## In CI diff --git a/internal/cli/environment.go b/internal/cli/environment.go index cd2cb00..581d694 100644 --- a/internal/cli/environment.go +++ b/internal/cli/environment.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" "github.com/steadybit/cli/v6/internal/environment" + "github.com/steadybit/cli/v6/internal/gitops" "github.com/steadybit/cli/v6/internal/platform" "github.com/steadybit/cli/v6/internal/resource" ) @@ -48,6 +49,7 @@ func newEnvironment() *cobra.Command { RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return environment.Apply(ctx, c, a) }), } fileFlags(apply, &a.Files, &a.Recursive, "environment") + dryRun(apply, gitops.Environment, &a.Files, &a.Recursive) var d environment.DeleteOptions del := &cobra.Command{ @@ -90,6 +92,6 @@ func newEnvironment() *cobra.Command { vset.Flags().BoolVar(&vs.Replace, "replace", false, "Remove every variable not given.") variable.AddCommand(vget, vset) - cmd.AddCommand(list, get, apply, del, variable) + cmd.AddCommand(list, get, apply, newDiff(gitops.Environment, "environment", "environment.yml", "environments"), del, variable) return cmd } diff --git a/internal/cli/experiment.go b/internal/cli/experiment.go index 13ed70f..267173f 100644 --- a/internal/cli/experiment.go +++ b/internal/cli/experiment.go @@ -19,7 +19,7 @@ import ( func newExperiment() *cobra.Command { cmd := &cobra.Command{Use: "experiment", Short: "Check and run experiments."} cmd.AddCommand(newExperimentRun(), newExperimentGet(), newExperimentApply(), newExperimentDelete(), newExperimentDump(), newExperimentInit(), - newDiff(gitops.Experiment, "experiment", "experiment.yml")) + newDiff(gitops.Experiment, "experiment", "experiment.yml", "experiments")) return cmd } diff --git a/internal/cli/gitops.go b/internal/cli/gitops.go index 58b599a..4087c24 100644 --- a/internal/cli/gitops.go +++ b/internal/cli/gitops.go @@ -13,7 +13,7 @@ import ( ) // newDiff is the `diff` command of a kind of file: `experiment diff`, `schedule diff`... -func newDiff(k gitops.Kind, group, example string) *cobra.Command { +func newDiff(k gitops.Kind, group, example, dir string) *cobra.Command { var files []string var recursive bool cmd := &cobra.Command{ @@ -22,7 +22,7 @@ func newDiff(k gitops.Kind, group, example string) *cobra.Command { Args: cobra.NoArgs, Example: examples( "steadybit "+group+" diff -f "+example, - "steadybit "+group+" diff -f ./"+group+"s -R || echo drift", + "steadybit "+group+" diff -f ./"+dir+" -R || echo drift", ), RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return gitops.DiffFiles(ctx, c, k, files, recursive) @@ -57,16 +57,19 @@ func newExport() *cobra.Command { var o gitops.ExportOptions cmd := &cobra.Command{ Use: "export", - Short: "Write a team's experiments, schedules, services and the custom service profiles they use to a directory, to keep in Git.", + Short: "Write a team's experiments, schedules, services and the custom service profiles they use to a directory, to keep in Git. With --tenant, write the tenant's experiment templates, environments, teams, property definitions, hubs, integrations and custom service profiles instead.", Args: cobra.NoArgs, Example: examples( "steadybit export --team ADM -d ./chaos", + "steadybit export --tenant -d ./platform", ), RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return gitops.Export(ctx, c, o) }), } cmd.Flags().StringVar(&o.Team, "team", "", "The key of the team to export.") + cmd.Flags().BoolVar(&o.Tenant, "tenant", false, "Export the configuration of the tenant rather than a team. Credentials of integrations are written masked.") cmd.Flags().StringVarP(&o.Directory, "directory", "d", ".", "The directory to write the project to.") - _ = cmd.MarkFlagRequired("team") + cmd.MarkFlagsOneRequired("team", "tenant") + cmd.MarkFlagsMutuallyExclusive("team", "tenant") return cmd } @@ -74,11 +77,12 @@ func newApplyProject() *cobra.Command { var o gitops.ApplyOptions cmd := &cobra.Command{ Use: "apply", - Short: "Apply a project written by `export`: service profiles, then services, experiments and schedules.", + Short: "Apply a project written by `export`: property definitions, environments, teams, hubs, experiment templates, integrations and service profiles, then services, experiments and schedules.", Args: cobra.NoArgs, Example: examples( "steadybit apply -d ./chaos --dry-run", "steadybit apply -d ./chaos", + "steadybit apply -d ./platform", ), RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return gitops.ApplyProject(ctx, c, o) }), } @@ -94,7 +98,7 @@ func newDiffProject() *cobra.Command { Use: "diff", Short: "Show how a project written by `export` differs from the platform. Exits with 2 when it does.", Args: cobra.NoArgs, - Example: examples("steadybit diff -d ./chaos"), + Example: examples("steadybit diff -d ./chaos", "steadybit diff -d ./platform"), RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return gitops.DiffProject(ctx, c, dir) }), diff --git a/internal/cli/hub.go b/internal/cli/hub.go index 8d7a3a4..1405457 100644 --- a/internal/cli/hub.go +++ b/internal/cli/hub.go @@ -7,6 +7,7 @@ import ( "context" "github.com/spf13/cobra" + "github.com/steadybit/cli/v6/internal/gitops" "github.com/steadybit/cli/v6/internal/hub" "github.com/steadybit/cli/v6/internal/platform" "github.com/steadybit/cli/v6/internal/resource" @@ -45,6 +46,7 @@ func newHub() *cobra.Command { RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return hub.Apply(ctx, c, a) }), } fileFlags(apply, &a.Files, &a.Recursive, "hub") + dryRun(apply, gitops.Hub, &a.Files, &a.Recursive) apply.Flags().BoolVar(&a.Synchronize, "synchronize", false, "Fetch the hub's templates from its repository, waiting until it is done.") var d hub.DeleteOptions @@ -69,6 +71,6 @@ func newHub() *cobra.Command { } idFlag(resync, &resyncID, "The hub id.") - cmd.AddCommand(list, get, apply, del, resync) + cmd.AddCommand(list, get, apply, newDiff(gitops.Hub, "hub", "hub.yml", "hubs"), del, resync) return cmd } diff --git a/internal/cli/integration.go b/internal/cli/integration.go index 9e24b6a..b616873 100644 --- a/internal/cli/integration.go +++ b/internal/cli/integration.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/spf13/cobra" + "github.com/steadybit/cli/v6/internal/gitops" "github.com/steadybit/cli/v6/internal/integration" "github.com/steadybit/cli/v6/internal/platform" "github.com/steadybit/cli/v6/internal/resource" @@ -67,6 +68,7 @@ func newIntegrationKind(k integration.Kind) *cobra.Command { }), } fileFlags(apply, &a.Files, &a.Recursive, lower) + dryRun(apply, gitops.Integrations[k.Name], &a.Files, &a.Recursive) var d integration.DeleteOptions del := &cobra.Command{ @@ -81,6 +83,6 @@ func newIntegrationKind(k integration.Kind) *cobra.Command { idFlag(del, &d.ID, "The "+lower+" id.") del.Flags().BoolVar(&d.Yes, "yes", false, yesHelp) - cmd.AddCommand(list, get, apply, del) + cmd.AddCommand(list, get, apply, newDiff(gitops.Integrations[k.Name], "integration "+k.Name, k.Name+".yml", "integrations/"+k.Name), del) return cmd } diff --git a/internal/cli/property.go b/internal/cli/property.go index 2a63df7..155a46f 100644 --- a/internal/cli/property.go +++ b/internal/cli/property.go @@ -7,6 +7,7 @@ import ( "context" "github.com/spf13/cobra" + "github.com/steadybit/cli/v6/internal/gitops" "github.com/steadybit/cli/v6/internal/platform" "github.com/steadybit/cli/v6/internal/property" "github.com/steadybit/cli/v6/internal/resource" @@ -65,6 +66,7 @@ func newPropertyDefinition() *cobra.Command { } fileFlags(apply, &a.Files, &a.Recursive, "property definition") apply.Flags().BoolVar(&a.DeleteValues, "delete-values", false, "Allow removing enum values still in use, deleting them where they are used.") + dryRun(apply, gitops.PropertyDefinition, &a.Files, &a.Recursive) var d property.DeleteDefinitionOptions del := &cobra.Command{ @@ -80,7 +82,7 @@ func newPropertyDefinition() *cobra.Command { del.Flags().BoolVar(&d.Associations, "delete-associations", false, "Also delete the associations of the property.") del.Flags().BoolVar(&d.Yes, "yes", false, yesHelp) - cmd.AddCommand(list, get, apply, del) + cmd.AddCommand(list, get, apply, newDiff(gitops.PropertyDefinition, "property definition", "result-color.yml", "property-definitions"), del) return cmd } diff --git a/internal/cli/schedule.go b/internal/cli/schedule.go index 958b1af..23f5ad4 100644 --- a/internal/cli/schedule.go +++ b/internal/cli/schedule.go @@ -126,7 +126,7 @@ func newSchedule() *cobra.Command { scheduleIDFlag(c, &id) return c } - cmd.AddCommand(list, get, apply, newDiff(gitops.Schedule, "schedule", "schedule.yml"), create, update, + cmd.AddCommand(list, get, apply, newDiff(gitops.Schedule, "schedule", "schedule.yml", "schedules"), create, update, idCommand("enable", "Enable an experiment schedule.", func(ctx context.Context, c *platform.Client, id string) error { return schedule.SetEnabled(ctx, c, id, true) }), diff --git a/internal/cli/service.go b/internal/cli/service.go index 1bc010f..46cadde 100644 --- a/internal/cli/service.go +++ b/internal/cli/service.go @@ -198,7 +198,7 @@ func newService() *cobra.Command { vset.Flags().BoolVar(&vs.Replace, "replace", false, "Remove every variable not given.") variable.AddCommand(vget, vset) - cmd.AddCommand(list, get, apply, newDiff(gitops.Service, "service", "service.yml"), del, risk, experiments, variable) + cmd.AddCommand(list, get, apply, newDiff(gitops.Service, "service", "service.yml", "services"), del, risk, experiments, variable) return cmd } @@ -260,6 +260,6 @@ func newServiceProfile() *cobra.Command { } idFlag(del, &deleteID, "The service profile id.") - cmd.AddCommand(list, get, apply, newDiff(gitops.ServiceProfile, "service-profile", "profile.yml"), del) + cmd.AddCommand(list, get, apply, newDiff(gitops.ServiceProfile, "service-profile", "profile.yml", "service-profiles"), del) return cmd } diff --git a/internal/cli/team.go b/internal/cli/team.go index cfc63a2..3f4f40b 100644 --- a/internal/cli/team.go +++ b/internal/cli/team.go @@ -7,6 +7,7 @@ import ( "context" "github.com/spf13/cobra" + "github.com/steadybit/cli/v6/internal/gitops" "github.com/steadybit/cli/v6/internal/platform" "github.com/steadybit/cli/v6/internal/resource" "github.com/steadybit/cli/v6/internal/team" @@ -51,6 +52,7 @@ func newTeam() *cobra.Command { RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return team.Apply(ctx, c, a) }), } fileFlags(apply, &a.Files, &a.Recursive, "team") + dryRun(apply, gitops.Team, &a.Files, &a.Recursive) var d team.DeleteOptions del := &cobra.Command{ @@ -64,7 +66,7 @@ func newTeam() *cobra.Command { del.Flags().BoolVar(&d.Experiments, "purge-experiments", false, "Also delete the team's experiments and their runs.") del.Flags().BoolVar(&d.Yes, "yes", false, yesHelp) - cmd.AddCommand(list, get, apply, del, newTeamMember(), newTeamEnvironment()) + cmd.AddCommand(list, get, apply, newDiff(gitops.Team, "team", "team.yml", "teams"), del, newTeamMember(), newTeamEnvironment()) return cmd } diff --git a/internal/cli/template.go b/internal/cli/template.go index 5d28dbe..ebd3a0b 100644 --- a/internal/cli/template.go +++ b/internal/cli/template.go @@ -7,6 +7,7 @@ import ( "context" "github.com/spf13/cobra" + "github.com/steadybit/cli/v6/internal/gitops" "github.com/steadybit/cli/v6/internal/platform" "github.com/steadybit/cli/v6/internal/resource" "github.com/steadybit/cli/v6/internal/template" @@ -80,6 +81,7 @@ func newTemplate() *cobra.Command { RunE: withClient(func(ctx context.Context, c *platform.Client, _ []string) error { return template.Apply(ctx, c, a) }), } fileFlags(apply, &a.Files, &a.Recursive, "template") + dryRun(apply, gitops.Template, &a.Files, &a.Recursive) var d template.DeleteOptions del := &cobra.Command{ @@ -107,6 +109,6 @@ func newTemplate() *cobra.Command { _ = imp.MarkFlagRequired("template") variadic(imp, "template") - cmd.AddCommand(list, get, apply, del, imp) + cmd.AddCommand(list, get, apply, newDiff(gitops.Template, "template", "template.yml", "templates"), del, imp) return cmd } diff --git a/internal/environment/environment.go b/internal/environment/environment.go index d9574ac..36f687e 100644 --- a/internal/environment/environment.go +++ b/internal/environment/environment.go @@ -19,7 +19,7 @@ import ( // The state is the platform's, and the version is dropped as `service get` drops it: kept // in a file, it turns every apply after an edit in the UI into a conflict. -var readOnly = []string{"version", "state"} +var ReadOnly = []string{"version", "state"} func uuid(id string) (openapi_types.UUID, error) { u, ok := resource.UUID(id) @@ -88,7 +88,7 @@ func Get(ctx context.Context, c *platform.Client, o GetOptions) error { if err != nil { return notFoundOr(err, o.ID, "Failed to get environment %s") } - if err := resource.Output(resource.Strip(doc, readOnly...), o.File, o.Type); err != nil { + if err := resource.Output(resource.Strip(doc, ReadOnly...), o.File, o.Type); err != nil { return err } if o.File != "" { @@ -109,7 +109,7 @@ func Apply(ctx context.Context, c *platform.Client, o ApplyOptions) error { return resource.Applied{}, fmt.Errorf("Environment file '%s' does not name the environment.", file) } var saved struct{ ID, Name string } - resp, err := c.UpsertEnvironmentWithBody(ctx, "application/json", resource.Body(resource.Strip(doc, readOnly...).Value())) + resp, err := c.UpsertEnvironmentWithBody(ctx, "application/json", resource.Body(resource.Strip(doc, ReadOnly...).Value())) resp, err = platform.Decode(resp, err, &saved) if err != nil { return resource.Applied{}, platform.Failed(err, "Failed to save environment %s", name) diff --git a/internal/gitops/kinds.go b/internal/gitops/kinds.go index c90ec21..1e9c252 100644 --- a/internal/gitops/kinds.go +++ b/internal/gitops/kinds.go @@ -7,24 +7,35 @@ import ( "context" "fmt" "net/http" + "strings" openapi_types "github.com/oapi-codegen/runtime/types" "github.com/steadybit/cli/v6/api" + "github.com/steadybit/cli/v6/internal/environment" + "github.com/steadybit/cli/v6/internal/hub" + "github.com/steadybit/cli/v6/internal/integration" "github.com/steadybit/cli/v6/internal/jsyaml" "github.com/steadybit/cli/v6/internal/output" "github.com/steadybit/cli/v6/internal/platform" + "github.com/steadybit/cli/v6/internal/property" + "github.com/steadybit/cli/v6/internal/team" + "github.com/steadybit/cli/v6/internal/template" ) // Kind is a type of file kept in Git and how to find its counterpart on the platform. type Kind struct { // Name is what messages call it, "experiment" or "service profile". Name string - // ReadOnly fields are reported by the platform but not part of the file. + // ReadOnly fields are reported by the platform but not part of the file. In + // "members.name", the field is dropped from each member. ReadOnly []string - // Defaults are the values the platform gives fields a file leaves out. Holding - // exactly that, such a field is not a difference; holding anything else, applying - // the file would reset it, which is. + // Defaults are the values the platform gives fields a file leaves out, or leaves as + // an empty list or map. Holding exactly that, such a field is not a difference; + // holding anything else, applying the file would reset it, which is. Defaults map[string]any + // Secrets are fields holding credentials, as integration.Kind names them. A mask in + // a file stands for whatever the platform holds, and no value is ever printed. + Secrets []string // Identity is the field that names it on the platform. A file matched otherwise, by // an externalId or a name, has none yet, which is not a difference. Identity string @@ -157,14 +168,159 @@ var ServiceProfile = Kind{ }, } +// byID finds the platform's version of a file by the id in it. +func byID(get func(ctx context.Context, c *platform.Client, id openapi_types.UUID) (*http.Response, error)) func(context.Context, *platform.Client, *jsyaml.Map) (string, *jsyaml.Map, error) { + return func(ctx context.Context, c *platform.Client, local *jsyaml.Map) (string, *jsyaml.Map, error) { + id := str(local, "id") + u, ok := uuid(id) + if !ok { + return "", nil, nil + } + remote, err := fetch(get(ctx, c, u)) + return id, remote, err + } +} + +var Template = Kind{ + Name: "experiment template", + ReadOnly: template.ReadOnly, + Identity: "id", + Remote: byID(func(ctx context.Context, c *platform.Client, id openapi_types.UUID) (*http.Response, error) { + return c.GetExperimentTemplate(ctx, id) + }), +} + +var Environment = Kind{ + Name: "environment", + ReadOnly: environment.ReadOnly, + Identity: "id", + // By its id, or by its name, which the platform matches a file without an id by. + Remote: func(ctx context.Context, c *platform.Client, local *jsyaml.Map) (string, *jsyaml.Map, error) { + if id := str(local, "id"); id != "" { + return byID(func(ctx context.Context, c *platform.Client, id openapi_types.UUID) (*http.Response, error) { + return c.GetEnvironment(ctx, id) + })(ctx, c, local) + } + name := str(local, "name") + if name == "" { + return "", nil, nil + } + var list struct { + Environments []struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"environments"` + } + resp, err := c.GetEnvironments(ctx, &api.GetEnvironmentsParams{Search: &name}) + if _, err := platform.Decode(resp, err, &list); err != nil { + return "", nil, err + } + for _, e := range list.Environments { + if e.Name == name { + u, _ := uuid(e.ID) + remote, err := fetch(c.GetEnvironment(ctx, u)) + return e.ID, remote, err + } + } + return "", nil, nil + }, +} + +// Team is matched by its key; the id `get` writes is not needed, so a file without one +// is not a difference. +var Team = Kind{ + Name: "team", + ReadOnly: append(append([]string{}, team.ReadOnly...), prefixed("members.", team.MemberReadOnly)...), + // Sent none, a team may still wait and validate services. + Defaults: map[string]any{"allowedActions": []any{"wait", "service-validation"}, "managedBy": "MANUAL"}, + Identity: "id", + Remote: func(ctx context.Context, c *platform.Client, local *jsyaml.Map) (string, *jsyaml.Map, error) { + key := str(local, "key") + if key == "" { + return "", nil, nil + } + remote, err := fetch(c.GetTeam(ctx, key)) + return key, remote, err + }, +} + +var PropertyDefinition = Kind{ + Name: "property definition", + ReadOnly: property.ReadOnly, + Identity: "key", + Remote: func(ctx context.Context, c *platform.Client, local *jsyaml.Map) (string, *jsyaml.Map, error) { + key := str(local, "key") + if key == "" { + return "", nil, nil + } + remote, err := fetch(c.GetPropertyDefinition(ctx, key)) + return key, remote, err + }, +} + +var Hub = Kind{ + Name: "hub", + ReadOnly: hub.ReadOnly, + Identity: "id", + Remote: byID(func(ctx context.Context, c *platform.Client, id openapi_types.UUID) (*http.Response, error) { + return c.GetHubById(ctx, id) + }), +} + +// Integrations are the kinds of integration by their command's name. +var Integrations = map[string]Kind{ + integration.Webhook.Name: integrationKind(integration.Webhook, "webhook integration", allTargetAttributes), + integration.Slack.Name: integrationKind(integration.Slack, "Slack integration", nil), + integration.Preflight.Name: integrationKind(integration.Preflight, "preflight webhook", allTargetAttributes), + integration.PreflightAction.Name: integrationKind(integration.PreflightAction, "preflight action integration", nil), +} + +// Sent none, a webhook reports every target attribute. +var allTargetAttributes = map[string]any{"targetAttributeIncludes": []any{"*"}} + +func integrationKind(k integration.Kind, name string, defaults map[string]any) Kind { + return Kind{ + Name: name, + ReadOnly: integration.ReadOnly, + Defaults: defaults, + Secrets: k.Secrets, + Identity: "id", + Remote: byID(k.Fetch), + } +} + +func prefixed(prefix string, fields []string) []string { + out := make([]string, len(fields)) + for i, f := range fields { + out[i] = prefix + f + } + return out +} + func strip(m *jsyaml.Map, fields []string) *jsyaml.Map { c := jsyaml.Clone(m).(*jsyaml.Map) for _, f := range fields { - c.Delete(f) + deletePath(c, strings.Split(f, ".")) } return c } +// deletePath removes a field, going into every item of the lists on its way. +func deletePath(v any, path []string) { + switch x := v.(type) { + case *jsyaml.Map: + if len(path) == 1 { + x.Delete(path[0]) + } else if child, ok := x.Get(path[0]); ok { + deletePath(child, path[1:]) + } + case []any: + for _, item := range x { + deletePath(item, path) + } + } +} + // State is what applying a file would do. type State int @@ -200,7 +356,9 @@ func Compare(ctx context.Context, c *platform.Client, k Kind, file string, local for key := range k.Defaults { defaulted = append(defaulted, key) } - diff, err := Diff(file, strip(local.Value(), ignored), withoutDefaults(local.Value(), strip(remote, ignored), k.Defaults), defaulted...) + l, r := strip(local.Value(), ignored), withoutDefaults(local.Value(), strip(remote, ignored), k.Defaults) + hideSecrets(l, r, k.Secrets) + diff, err := Diff(file, l, r, defaulted...) if err != nil { return Result{}, err } @@ -221,16 +379,76 @@ func (r Result) Describe(k Kind) string { return fmt.Sprintf("%s matches %s %s.", r.File, k.Name, r.ID) } -// withoutDefaults drops the fields the file leaves out that hold the platform's default. +// withoutDefaults drops the fields the file leaves out that hold the platform's default, +// and takes the file's empty list or map for the default the platform turned it into. func withoutDefaults(local, remote *jsyaml.Map, defaults map[string]any) *jsyaml.Map { out := jsyaml.Clone(remote).(*jsyaml.Map) for key, value := range defaults { - if _, set := local.Get(key); set { + held, ok := out.Get(key) + if !ok || jsyaml.CompactJSON(held) != jsyaml.CompactJSON(value) { continue } - if held, ok := out.Get(key); ok && jsyaml.CompactJSON(held) == jsyaml.CompactJSON(value) { + switch set, has := local.Get(key); { + case !has: out.Delete(key) + case isEmptyCollection(set): + out.Set(key, set) } } return out } + +func isEmptyCollection(v any) bool { + switch x := v.(type) { + case []any: + return len(x) == 0 + case *jsyaml.Map: + return x.Len() == 0 + } + return false +} + +// hideSecrets makes credentials comparable without printing them. The platform's value +// shows as the mask, which a mask in the file matches, as does the value itself; any +// other value in the file is what applying it would set, and shows as that. +func hideSecrets(local, remote *jsyaml.Map, secrets []string) { + for _, field := range secrets { + lv, _ := local.Get(field) + rv, _ := remote.Get(field) + lm, lIsMap := lv.(*jsyaml.Map) + rm, rIsMap := rv.(*jsyaml.Map) + if lIsMap || rIsMap { + if lm == nil { + lm = jsyaml.NewMap() + } + if rm == nil { + rm = jsyaml.NewMap() + } + for _, key := range lm.Keys() { + hideSecret(lm, rm, key) + } + for _, key := range rm.Keys() { + hideSecret(lm, rm, key) + } + continue + } + hideSecret(local, remote, field) + } +} + +func hideSecret(local, remote *jsyaml.Map, key string) { + lv, lok := local.Get(key) + rv, rok := remote.Get(key) + rs, _ := rv.(string) + ls, _ := lv.(string) + if rok && rs != "" { + remote.Set(key, integration.Mask) + } + switch { + case !lok || ls == "" || ls == integration.Mask: + case integration.Masked(ls) || (rok && ls == rs): + local.Set(key, integration.Mask) + default: + local.Set(key, integration.Mask+" (from the file)") + } +} diff --git a/internal/gitops/project.go b/internal/gitops/project.go index 83126b8..c32ca8e 100644 --- a/internal/gitops/project.go +++ b/internal/gitops/project.go @@ -15,26 +15,87 @@ import ( "strings" "github.com/steadybit/cli/v6/api" + "github.com/steadybit/cli/v6/internal/environment" "github.com/steadybit/cli/v6/internal/experiment" + "github.com/steadybit/cli/v6/internal/hub" + "github.com/steadybit/cli/v6/internal/integration" "github.com/steadybit/cli/v6/internal/jsyaml" "github.com/steadybit/cli/v6/internal/output" "github.com/steadybit/cli/v6/internal/platform" + "github.com/steadybit/cli/v6/internal/property" "github.com/steadybit/cli/v6/internal/schedule" "github.com/steadybit/cli/v6/internal/service" "github.com/steadybit/cli/v6/internal/serviceprofile" + "github.com/steadybit/cli/v6/internal/team" + "github.com/steadybit/cli/v6/internal/template" ) -// A project is a directory holding what a team keeps in Git, one kind per directory, -// in the order applying them has to follow: services name their profile, schedules -// their experiment. +// A project is a directory holding what a team or the tenant keeps in Git, one kind per +// directory, in the order applying them has to follow. Templates and experiments carry +// the properties definitions define; teams name their environments, integrations their +// team, service profiles their templates, services their profile, experiments their +// team and environment, schedules their experiment. Hubs depend on nothing. var projectKinds = []struct { - dir string - kind Kind + dir string + kind Kind + apply func(ctx context.Context, c *platform.Client, path string, o ApplyOptions) error }{ - {"service-profiles", ServiceProfile}, - {"services", Service}, - {"experiments", Experiment}, - {"schedules", Schedule}, + {"property-definitions", PropertyDefinition, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + return property.ApplyDefinitions(ctx, c, property.ApplyDefinitionOptions{Files: []string{path}, Recursive: true}) + }}, + {"environments", Environment, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + return environment.Apply(ctx, c, environment.ApplyOptions{Files: []string{path}, Recursive: true}) + }}, + {"teams", Team, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + return team.Apply(ctx, c, team.ApplyOptions{Files: []string{path}, Recursive: true}) + }}, + {"hubs", Hub, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + return hub.Apply(ctx, c, hub.ApplyOptions{Files: []string{path}, Recursive: true}) + }}, + {"templates", Template, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + return template.Apply(ctx, c, template.ApplyOptions{Files: []string{path}, Recursive: true}) + }}, + {"integrations/webhook", Integrations[integration.Webhook.Name], applyIntegrations(integration.Webhook)}, + {"integrations/slack", Integrations[integration.Slack.Name], applyIntegrations(integration.Slack)}, + {"integrations/preflight", Integrations[integration.Preflight.Name], applyIntegrations(integration.Preflight)}, + {"integrations/preflight-action", Integrations[integration.PreflightAction.Name], applyIntegrations(integration.PreflightAction)}, + {"service-profiles", ServiceProfile, func(ctx context.Context, c *platform.Client, path string, o ApplyOptions) error { + return serviceprofile.Apply(ctx, c, serviceprofile.ApplyOptions{Files: []string{path}, Recursive: true, DeleteExperiments: o.DeleteExperiments}) + }}, + {"services", Service, func(ctx context.Context, c *platform.Client, path string, o ApplyOptions) error { + return service.Apply(ctx, c, service.ApplyOptions{Files: []string{path}, Recursive: true, DeleteExperiments: o.DeleteExperiments}) + }}, + {"experiments", Experiment, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + return experiment.Apply(ctx, c, experiment.ApplyOptions{Files: []string{path}, Recursive: true}) + }}, + {"schedules", Schedule, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + return schedule.Apply(ctx, c, schedule.ApplyOptions{Files: []string{path}, Recursive: true}) + }}, +} + +// applyIntegrations leaves out the files that match the platform. Those holding a masked +// secret could not be applied, the platform keeping no secret it is not sent, and the +// others need not be. +func applyIntegrations(k integration.Kind) func(ctx context.Context, c *platform.Client, path string, o ApplyOptions) error { + return func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { + gk := Integrations[k.Name] + results, err := compareAll(ctx, c, gk, []string{path}, true) + if err != nil { + return err + } + var changed []string + for _, r := range results { + if r.State == Unchanged { + fmt.Println(r.Describe(gk)) + } else { + changed = append(changed, r.File) + } + } + if len(changed) == 0 { + return nil + } + return integration.Apply(ctx, c, k, integration.ApplyOptions{Files: changed}) + } } var unsafe = regexp.MustCompile(`[^a-z0-9._-]+`) @@ -63,12 +124,17 @@ func writeDocument(file string, doc *jsyaml.Map, readOnly []string) error { type ExportOptions struct { Directory string Team string + Tenant bool } // Export writes a team's experiments, schedules and services, and the custom service -// profiles those services use, as a project. Files are only written, never removed, so -// something deleted on the platform keeps its file until it is removed by hand. +// profiles those services use, as a project; or the tenant's configuration. Files are +// only written, never removed, so something deleted on the platform keeps its file until +// it is removed by hand. func Export(ctx context.Context, c *platform.Client, o ExportOptions) error { + if o.Tenant { + return exportTenant(ctx, c, o) + } taken := map[string]bool{} counts := map[string]int{} team := []string{o.Team} @@ -169,14 +235,16 @@ func mapWith(key, value string) *jsyaml.Map { // projectDirs are the kinds a project directory holds, in the order to apply them. func projectDirs(dir string) (map[string]string, error) { found := map[string]string{} + var names []string for _, pk := range projectKinds { - path := filepath.Join(dir, pk.dir) + names = append(names, pk.dir+"/") + path := filepath.Join(dir, filepath.FromSlash(pk.dir)) if info, err := os.Stat(path); err == nil && info.IsDir() { found[pk.dir] = path } } if len(found) == 0 { - return nil, fmt.Errorf("'%s' holds none of experiments/, schedules/, services/ or service-profiles/.", dir) + return nil, fmt.Errorf("'%s' holds none of %s or %s.", dir, strings.Join(names[:len(names)-1], ", "), names[len(names)-1]) } return found, nil } @@ -187,7 +255,7 @@ type ApplyOptions struct { DryRun bool } -// ApplyProject applies every kind in a project, profiles first and schedules last. +// ApplyProject applies every kind in a project, in the order of projectKinds. func ApplyProject(ctx context.Context, c *platform.Client, o ApplyOptions) error { dirs, err := projectDirs(o.Directory) if err != nil { @@ -201,16 +269,7 @@ func ApplyProject(ctx context.Context, c *platform.Client, o ApplyOptions) error if o.DryRun { err = DryRun(ctx, c, pk.kind, []string{path}, true) } else { - switch pk.dir { - case "service-profiles": - err = serviceprofile.Apply(ctx, c, serviceprofile.ApplyOptions{Files: []string{path}, Recursive: true, DeleteExperiments: o.DeleteExperiments}) - case "services": - err = service.Apply(ctx, c, service.ApplyOptions{Files: []string{path}, Recursive: true, DeleteExperiments: o.DeleteExperiments}) - case "experiments": - err = experiment.Apply(ctx, c, experiment.ApplyOptions{Files: []string{path}, Recursive: true}) - case "schedules": - err = schedule.Apply(ctx, c, schedule.ApplyOptions{Files: []string{path}, Recursive: true}) - } + err = pk.apply(ctx, c, path, o) } if err != nil { return err diff --git a/internal/gitops/project_test.go b/internal/gitops/project_test.go index 9eb2c6c..4f9243c 100644 --- a/internal/gitops/project_test.go +++ b/internal/gitops/project_test.go @@ -79,5 +79,5 @@ func TestAProjectEditedLocallyDrifts(t *testing.T) { func TestApplyNeedsAProject(t *testing.T) { err := gitops.ApplyProject(ctx, nil, gitops.ApplyOptions{Directory: t.TempDir()}) - assert.ErrorContains(t, err, "holds none of experiments/, schedules/, services/ or service-profiles/.") + assert.ErrorContains(t, err, "holds none of property-definitions/, environments/, teams/, hubs/, templates/, integrations/webhook/, integrations/slack/, integrations/preflight/, integrations/preflight-action/, service-profiles/, services/, experiments/ or schedules/.") } diff --git a/internal/gitops/tenant.go b/internal/gitops/tenant.go new file mode 100644 index 0000000..d15b625 --- /dev/null +++ b/internal/gitops/tenant.go @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2026 Steadybit GmbH + +package gitops + +import ( + "context" + "fmt" + "net/http" + "path/filepath" + + openapi_types "github.com/oapi-codegen/runtime/types" + "github.com/steadybit/cli/v6/api" + "github.com/steadybit/cli/v6/internal/integration" + "github.com/steadybit/cli/v6/internal/jsyaml" + "github.com/steadybit/cli/v6/internal/platform" +) + +// The hubs every tenant is connected to by the platform itself, with the same ids +// everywhere. Kept in Git, they would be connected a second time on another platform. +var providedHubs = map[string]bool{ + "6a55640d-72d4-4194-a058-0afcf731dfac": true, // Steadybit Reliability Hub + "738c90be-bdc4-4e0b-a4de-c8f1144b25c7": true, // Steadybit Service Templates +} + +// tenantExport writes one kind of the tenant's configuration. +type tenantExport struct { + o ExportOptions + taken map[string]bool + counts map[string]int +} + +func (e *tenantExport) write(dir, name string, doc *jsyaml.Map, k Kind) error { + e.counts[dir]++ + return writeDocument(fileName(filepath.Join(e.o.Directory, dir), name, e.taken), doc, k.ReadOnly) +} + +func (e *tenantExport) each(ids []string, what string, get func(id string) (*http.Response, error), write func(doc *jsyaml.Map) error) error { + for _, id := range ids { + doc, _, err := platform.ReadDocument(get(id)) + if err != nil { + return platform.Failed(err, "Failed to get %s %s", what, id) + } + if err := write(doc.Value()); err != nil { + return err + } + } + return nil +} + +func byUUID(get func(openapi_types.UUID) (*http.Response, error)) func(string) (*http.Response, error) { + return func(id string) (*http.Response, error) { + u, _ := uuid(id) + return get(u) + } +} + +// exportTenant writes what the tenant's admins configure. What the platform provides +// is left out, as it comes with every platform: the hubs it connects, its service +// profiles, and the templates imported from a hub, which importing again brings back. +func exportTenant(ctx context.Context, c *platform.Client, o ExportOptions) error { + e := &tenantExport{o: o, taken: map[string]bool{}, counts: map[string]int{}} + + type keyed struct { + Key string `json:"key"` + } + definitions, err := platform.AllPages[keyed](func(page, size int32) (*http.Response, error) { + return c.GetPropertyDefinitions(ctx, &api.GetPropertyDefinitionsParams{Page: api.PageRequestAO{Page: &page, Size: &size}}) + }) + if err != nil { + return platform.Failed(err, "Failed to get the property definitions") + } + var keys []string + for _, d := range definitions { + keys = append(keys, d.Key) + } + if err := e.each(keys, "property definition", func(key string) (*http.Response, error) { return c.GetPropertyDefinition(ctx, key) }, + func(doc *jsyaml.Map) error { + return e.write("property-definitions", str(doc, "key"), doc, PropertyDefinition) + }); err != nil { + return err + } + + var environments struct { + Environments []struct { + ID string `json:"id"` + } `json:"environments"` + } + resp, err := c.GetEnvironments(ctx, &api.GetEnvironmentsParams{}) + if _, err := platform.Decode(resp, err, &environments); err != nil { + return platform.Failed(err, "Failed to get the environments") + } + var ids []string + for _, env := range environments.Environments { + ids = append(ids, env.ID) + } + if err := e.each(ids, "environment", byUUID(func(id openapi_types.UUID) (*http.Response, error) { return c.GetEnvironment(ctx, id) }), + func(doc *jsyaml.Map) error { return e.write("environments", str(doc, "name"), doc, Environment) }); err != nil { + return err + } + + var teams struct { + Teams []keyed `json:"teams"` + } + resp, err = c.GetTeams(ctx, &api.GetTeamsParams{}) + if _, err := platform.Decode(resp, err, &teams); err != nil { + return platform.Failed(err, "Failed to get the teams") + } + keys = nil + for _, t := range teams.Teams { + keys = append(keys, t.Key) + } + if err := e.each(keys, "team", func(key string) (*http.Response, error) { return c.GetTeam(ctx, key) }, + func(doc *jsyaml.Map) error { return e.write("teams", str(doc, "key"), doc, Team) }); err != nil { + return err + } + + var hubs struct { + Hubs []struct { + ID string `json:"id"` + } `json:"hubs"` + } + resp, err = c.GetHubs(ctx) + if _, err := platform.Decode(resp, err, &hubs); err != nil { + return platform.Failed(err, "Failed to get the hubs") + } + ids = nil + for _, h := range hubs.Hubs { + ids = append(ids, h.ID) + } + // An imported template keeps the id it has in its hub, which is how one is told. + imported := map[string]bool{} + if err := e.each(ids, "hub", byUUID(func(id openapi_types.UUID) (*http.Response, error) { return c.GetHubById(ctx, id) }), + func(doc *jsyaml.Map) error { + templates, _ := doc.Get("templates") + list, _ := templates.([]any) + for _, t := range list { + if m, ok := t.(*jsyaml.Map); ok { + imported[str(m, "id")] = true + } + } + if providedHubs[str(doc, "id")] { + return nil + } + return e.write("hubs", str(doc, "hubName"), doc, Hub) + }); err != nil { + return err + } + + var templates struct { + Templates []struct { + ID string `json:"id"` + } `json:"templates"` + } + resp, err = c.GetExperimentTemplates(ctx, &api.GetExperimentTemplatesParams{}) + if _, err := platform.Decode(resp, err, &templates); err != nil { + return platform.Failed(err, "Failed to get the experiment templates") + } + ids = nil + for _, t := range templates.Templates { + if !imported[t.ID] { + ids = append(ids, t.ID) + } + } + if err := e.each(ids, "experiment template", byUUID(func(id openapi_types.UUID) (*http.Response, error) { return c.GetExperimentTemplate(ctx, id) }), + func(doc *jsyaml.Map) error { return e.write("templates", str(doc, "templateTitle"), doc, Template) }); err != nil { + return err + } + + for _, k := range integration.Kinds { + var list struct { + Content []struct { + ID string `json:"id"` + } `json:"content"` + } + resp, err := k.FetchAll(ctx, c) + if _, err := platform.Decode(resp, err, &list); err != nil { + return platform.Failed(err, "Failed to get the %s", k.Plural) + } + ids = nil + for _, i := range list.Content { + ids = append(ids, i.ID) + } + dir := "integrations/" + k.Name + if err := e.each(ids, k.Title, byUUID(func(id openapi_types.UUID) (*http.Response, error) { return k.Fetch(ctx, c, id) }), + func(doc *jsyaml.Map) error { + k.MaskSecrets(doc) + e.counts["integrations"]++ + return e.write(dir, str(doc, "name"), doc, Integrations[k.Name]) + }); err != nil { + return err + } + } + + // Profiles Steadybit provides come with the platform. + type profile struct { + ID string `json:"id"` + Origin string `json:"origin"` + } + profiles, err := platform.AllPages[profile](func(page, size int32) (*http.Response, error) { + return c.GetProfiles(ctx, &api.GetProfilesParams{Page: api.PageRequestAO{Page: &page, Size: &size}}) + }) + if err != nil { + return platform.Failed(err, "Failed to get the service profiles") + } + ids = nil + for _, p := range profiles { + if p.Origin == "CUSTOM" { + ids = append(ids, p.ID) + } + } + if err := e.each(ids, "service profile", byUUID(func(id openapi_types.UUID) (*http.Response, error) { return c.GetProfile(ctx, id) }), + func(doc *jsyaml.Map) error { return e.write("service-profiles", str(doc, "name"), doc, ServiceProfile) }); err != nil { + return err + } + + fmt.Printf("Exported the tenant to %s: %d experiment templates, %d environments, %d teams, %d property definitions, %d hubs, %d integrations, %d service profiles.\n", + o.Directory, e.counts["templates"], e.counts["environments"], e.counts["teams"], e.counts["property-definitions"], e.counts["hubs"], + e.counts["integrations"], e.counts["service-profiles"]) + return nil +} diff --git a/internal/gitops/tenant_test.go b/internal/gitops/tenant_test.go new file mode 100644 index 0000000..0a12479 --- /dev/null +++ b/internal/gitops/tenant_test.go @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: 2026 Steadybit GmbH + +package gitops_test + +import ( + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steadybit/cli/v6/internal/gitops" + "github.com/steadybit/cli/v6/internal/platformtest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + environmentID = "019eacd7-fb2c-733a-bed5-99a935323d01" + teamID = "019eacd7-fb2c-733a-bed5-99a935323d02" + hubID = "019eacd7-fb2c-733a-bed5-99a935323d03" + templateID = "019eacd7-fb2c-733a-bed5-99a935323d04" + importedID = "019eacd7-fb2c-733a-bed5-99a935323d05" + webhookID = "019eacd7-fb2c-733a-bed5-99a935323d06" + slackID = "019eacd7-fb2c-733a-bed5-99a935323d07" + providedHubID = "6a55640d-72d4-4194-a058-0afcf731dfac" +) + +const ( + storedWebhook = `{"id":"` + webhookID + `","version":0,"scope":"GLOBAL","name":"Notify","url":"https://example.com/hook","secret":"****************","events":["*"],"targetAttributeIncludes":["*"],"headers":{"Authorization":"Bearer abc"}}` + storedSlack = `{"id":"` + slackID + `","version":1,"scope":"GLOBAL","name":"Chat","url":"https://hooks.example.com/T1/B2/xyz","events":["*"],"channel":"#chaos"}` +) + +func fakeTenant(t *testing.T) *platformtest.Platform { + p := platformtest.New(t) + p.Reply("GET /api/properties/definitions", platformtest.Reply{JSON: map[string]any{"items": []any{map[string]any{"key": "tribe"}}}}) + p.Reply("GET /api/properties/definitions/tribe", platformtest.Reply{Body: `{"key":"tribe","label":"Tribe","dataType":"STRING","version":3}`}) + p.Reply("GET /api/environments", platformtest.Reply{JSON: map[string]any{"environments": []any{map[string]any{"id": environmentID}}}}) + p.Reply("GET /api/environments/"+environmentID, platformtest.Reply{Body: `{"id":"` + environmentID + `","name":"Prod","version":2,"predicate":{"operator":"AND","predicates":[]},"state":"READY"}`}) + p.Reply("GET /api/teams", platformtest.Reply{JSON: map[string]any{"teams": []any{map[string]any{"key": "ADM"}}}}) + p.Reply("GET /api/teams/ADM", platformtest.Reply{Body: `{"id":"` + teamID + `","key":"ADM","name":"Admins","allowedActions":["wait","service-validation"],"allowedEnvironments":["Prod"],"managedBy":"MANUAL","version":0,` + + `"members":[{"username":"u1","name":"Jane","email":"jane@example.com","role":"OWNER","pictureUrl":"https://example.com/jane.png","managedBy":"MANUAL"}]}`}) + p.Reply("GET /api/hubs", platformtest.Reply{JSON: map[string]any{"hubs": []any{map[string]any{"id": providedHubID}, map[string]any{"id": hubID}}}}) + p.Reply("GET /api/hubs/"+providedHubID, platformtest.Reply{Body: `{"hubName":"Steadybit Reliability Hub","id":"` + providedHubID + `","templates":[{"id":"` + importedID + `","templateTitle":"Imported"}]}`}) + p.Reply("GET /api/hubs/"+hubID, platformtest.Reply{Body: `{"hubName":"Own Hub","repositoryUrl":"https://example.com/index.json","id":"` + hubID + `","version":4,"templates":[],"lastSync":"x","created":"c"}`}) + p.Reply("GET /api/experiments/templates", platformtest.Reply{JSON: map[string]any{"templates": []any{map[string]any{"id": templateID}, map[string]any{"id": importedID}}}}) + p.Reply("GET /api/experiments/templates/"+templateID, platformtest.Reply{Body: `{"id":"` + templateID + `","templateTitle":"Pod Crash","templateDescription":"d","placeholders":[],"tags":[],"lanes":[{"steps":[{"type":"wait","ignoreFailure":false,"parameters":{"duration":"5s"}}]}],"properties":{},"propertiesMetadata":[],"version":0,"created":"c","createdBy":{}}`}) + p.Reply("GET /api/integrations/webhook", platformtest.Reply{JSON: map[string]any{"content": []any{map[string]any{"id": webhookID}}}}) + p.Reply("GET /api/integrations/webhook/"+webhookID, platformtest.Reply{Body: storedWebhook}) + p.Reply("GET /api/integrations/slack", platformtest.Reply{JSON: map[string]any{"content": []any{map[string]any{"id": slackID}}}}) + p.Reply("GET /api/integrations/slack/"+slackID, platformtest.Reply{Body: storedSlack}) + p.Reply("GET /api/integrations/preflight", platformtest.Reply{JSON: map[string]any{"content": []any{}}}) + p.Reply("GET /api/integrations/preflight-action", platformtest.Reply{JSON: map[string]any{"content": []any{}}}) + p.Reply("GET /api/services/profiles", platformtest.Reply{JSON: map[string]any{"items": []any{ + map[string]any{"id": profileID, "name": "Shop", "origin": "CUSTOM"}, + map[string]any{"id": serviceID, "name": "Kubernetes Deployment", "origin": "STEADYBIT"}, + }}}) + p.Reply("GET /api/services/profiles/"+profileID, platformtest.Reply{Body: `{"id":"` + profileID + `","name":"Shop","origin":"CUSTOM","templates":[],"defaultProfile":false,"version":1}`}) + return p +} + +func TestExportTenantLeavesOutWhatThePlatformProvides(t *testing.T) { + p := fakeTenant(t) + dir := t.TempDir() + + out, err := platformtest.Stdout(t, func() error { return gitops.Export(ctx, p.Client, gitops.ExportOptions{Directory: dir, Tenant: true}) }) + + require.NoError(t, err) + assert.Equal(t, "Exported the tenant to "+dir+": 1 experiment templates, 1 environments, 1 teams, 1 property definitions, 1 hubs, 2 integrations, 1 service profiles.\n", out) + for file, content := range map[string]string{ + "property-definitions/tribe.yaml": "key: tribe\nlabel: Tribe\ndataType: STRING\n", + "environments/prod.yaml": "id: " + environmentID + "\nname: Prod\npredicate:\n operator: AND\n predicates: []\n", + "teams/adm.yaml": "id: " + teamID + "\nkey: ADM\nname: Admins\nallowedActions:\n - wait\n - service-validation\nallowedEnvironments:\n - Prod\nmanagedBy: MANUAL\n" + + "members:\n - username: u1\n email: jane@example.com\n role: OWNER\n", + "hubs/own-hub.yaml": "hubName: Own Hub\nrepositoryUrl: https://example.com/index.json\nid: " + hubID + "\n", + "templates/pod-crash.yaml": "id: " + templateID + "\ntemplateTitle: Pod Crash\ntemplateDescription: d\nplaceholders: []\ntags: []\nlanes:\n - steps:\n - type: wait\n ignoreFailure: false\n" + + " parameters:\n duration: 5s\nproperties: {}\npropertiesMetadata: []\n", + // Credentials are masked: the platform's mask of the secret gives away its length. + "integrations/webhook/notify.yaml": "id: " + webhookID + "\nscope: GLOBAL\nname: Notify\nurl: https://example.com/hook\nsecret: '********'\nevents:\n - '*'\n" + + "targetAttributeIncludes:\n - '*'\nheaders:\n Authorization: '********'\n", + "integrations/slack/chat.yaml": "id: " + slackID + "\nscope: GLOBAL\nname: Chat\nurl: '********'\nevents:\n - '*'\nchannel: '#chaos'\n", + "service-profiles/shop.yaml": "id: " + profileID + "\nname: Shop\norigin: CUSTOM\ntemplates: []\n", + } { + written, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(file))) + require.NoError(t, err, file) + assert.Equal(t, content, string(written), file) + } + for _, gone := range []string{"hubs/steadybit-reliability-hub.yaml", "templates/imported.yaml", "service-profiles/kubernetes-deployment.yaml", "integrations/preflight"} { + assert.NoFileExists(t, filepath.Join(dir, filepath.FromSlash(gone))) + } + assert.Empty(t, p.Requests("GET /api/experiments/templates/"+importedID), "a template imported from a hub is not even fetched") + + out, err = platformtest.Stdout(t, func() error { return gitops.DiffProject(ctx, p.Client, dir) }) + require.NoError(t, err) + assert.Equal(t, 8, strings.Count(out, "match the platform"), out) +} + +// Applying an exported tenant leaves out the integrations that match: the masked secret +// in them could not be sent back, and leaving it out would remove it. +func TestApplyingAnExportedTenantSkipsMatchingIntegrations(t *testing.T) { + p := fakeTenant(t) + dir := t.TempDir() + _, err := platformtest.Stdout(t, func() error { return gitops.Export(ctx, p.Client, gitops.ExportOptions{Directory: dir, Tenant: true}) }) + require.NoError(t, err) + for _, route := range []string{"POST /api/properties/definitions", "POST /api/environments", "POST /api/teams", "POST /api/hubs", "POST /api/experiments/templates", "POST /api/services/profiles"} { + p.Reply(route, platformtest.Reply{JSON: map[string]any{}}) + } + + out, err := platformtest.Stdout(t, func() error { return gitops.ApplyProject(ctx, p.Client, gitops.ApplyOptions{Directory: dir}) }) + + require.NoError(t, err) + assert.Contains(t, out, "notify.yaml matches webhook integration "+webhookID+".\n") + assert.Contains(t, out, "chat.yaml matches Slack integration "+slackID+".\n") + assert.Empty(t, p.Requests("POST /api/integrations/webhook")) + assert.Empty(t, p.Requests("POST /api/integrations/slack")) + order := []string{"Property definition tribe", "Environment", "Team ADM", "Hub", "Experiment template", "Service profile"} + last := -1 + for _, line := range order { + i := strings.Index(out, line) + assert.Greater(t, i, last, "%s is applied after what it depends on:\n%s", line, out) + last = i + } +} + +// A changed integration is applied with the credentials the platform reads back in the +// clear; its secret it never does, so that has to be put in. +func TestAChangedExportedIntegrationKeepsItsCredentials(t *testing.T) { + p := fakeTenant(t) + dir := t.TempDir() + _, err := platformtest.Stdout(t, func() error { return gitops.Export(ctx, p.Client, gitops.ExportOptions{Directory: dir, Tenant: true}) }) + require.NoError(t, err) + for _, other := range []string{"property-definitions", "environments", "teams", "hubs", "templates", "service-profiles", "integrations/webhook"} { + require.NoError(t, os.RemoveAll(filepath.Join(dir, filepath.FromSlash(other)))) + } + slack := filepath.Join(dir, "integrations", "slack", "chat.yaml") + require.NoError(t, os.WriteFile(slack, []byte("id: "+slackID+"\nscope: GLOBAL\nname: Chat\nurl: '********'\nevents:\n - '*'\nchannel: '#incidents'\n"), 0o644)) + p.Reply("POST /api/integrations/slack", platformtest.Reply{JSON: map[string]any{"id": slackID, "name": "Chat"}}) + + out, err := platformtest.Stdout(t, func() error { + return gitops.ApplyProject(ctx, p.Client, gitops.ApplyOptions{Directory: dir}) + }) + + require.NoError(t, err) + assert.Equal(t, "Slack integration Chat ("+slackID+") updated.\n", out) + sent := p.Requests("POST /api/integrations/slack")[0].JSON(t).(map[string]any) + assert.Equal(t, "https://hooks.example.com/T1/B2/xyz", sent["url"]) + assert.Equal(t, "#incidents", sent["channel"]) + written, _ := os.ReadFile(slack) + assert.Contains(t, string(written), "url: '********'\n", "the credential is not written to the file") +} + +func TestCredentialsAreNeverPrinted(t *testing.T) { + p := platformtest.New(t) + p.Reply("GET /api/integrations/webhook/"+webhookID, platformtest.Reply{Body: storedWebhook}) + file := write(t, "w.yml", "id: "+webhookID+"\nscope: GLOBAL\nname: Notify\nurl: https://example.com/hook\nsecret: n3w-s3cret\nevents:\n - '*'\n"+ + "targetAttributeIncludes:\n - '*'\nheaders:\n Authorization: Bearer other\n") + + out, err := platformtest.Stdout(t, func() error { + return gitops.DiffFiles(ctx, p.Client, gitops.Integrations["webhook"], []string{file}, false) + }) + + assert.ErrorIs(t, err, gitops.ErrDifferent) + assert.Contains(t, out, "-secret: '********'\n+secret: '******** (from the file)'\n") + assert.Contains(t, out, "- Authorization: '********'\n+ Authorization: '******** (from the file)'\n") + assert.NotContains(t, out, "s3cret") + assert.NotContains(t, out, "Bearer") +} + +// A team sent no actions may still wait and validate services, and a webhook sent no +// target attributes reports all of them. Neither is a difference. +func TestAHandWrittenTeamAndWebhookMatchRightAfterApply(t *testing.T) { + p := platformtest.New(t) + p.Reply("GET /api/teams/CLIX", platformtest.Reply{Body: `{"id":"` + teamID + `","key":"CLIX","name":"x","allowedActions":["wait","service-validation"],"allowedEnvironments":["Prod"],"managedBy":"MANUAL","version":0,"members":[]}`}) + p.Reply("GET /api/integrations/webhook/"+webhookID, platformtest.Reply{Body: `{"id":"` + webhookID + `","version":0,"scope":"GLOBAL","name":"n","url":"https://example.com","events":["*"],"targetAttributeIncludes":["*"],"headers":{}}`}) + team := write(t, "team.yml", "key: CLIX\nname: x\nallowedActions: []\nallowedEnvironments:\n - Prod\n") + webhook := write(t, "webhook.yml", "id: "+webhookID+"\nscope: GLOBAL\nname: n\nurl: https://example.com\nevents:\n - '*'\ntargetAttributeIncludes: []\n") + + _, err := platformtest.Stdout(t, func() error { return gitops.DiffFiles(ctx, p.Client, gitops.Team, []string{team}, false) }) + require.NoError(t, err) + _, err = platformtest.Stdout(t, func() error { + return gitops.DiffFiles(ctx, p.Client, gitops.Integrations["webhook"], []string{webhook}, false) + }) + require.NoError(t, err) + + p.Reply("GET /api/teams/CLIX", platformtest.Reply{Body: `{"key":"CLIX","name":"x","allowedActions":["wait"],"allowedEnvironments":["Prod"]}`}) + out, err := platformtest.Stdout(t, func() error { return gitops.DiffFiles(ctx, p.Client, gitops.Team, []string{team}, false) }) + assert.ErrorIs(t, err, gitops.ErrDifferent) + assert.Contains(t, out, "-allowedActions:\n- - wait\n+allowedActions: []\n") +} + +func TestEnvironmentsAreFoundByName(t *testing.T) { + p := platformtest.New(t) + p.Reply("GET /api/environments", platformtest.Reply{JSON: map[string]any{"environments": []any{ + map[string]any{"id": teamID, "name": "Prod EU"}, + map[string]any{"id": environmentID, "name": "Prod"}, + }}}) + p.Reply("GET /api/environments/"+environmentID, platformtest.Reply{Body: `{"id":"` + environmentID + `","name":"Prod","version":2,"predicate":{"operator":"AND","predicates":[]},"state":"READY"}`}) + file := write(t, "e.yml", "name: Prod\npredicate:\n operator: AND\n predicates: []\n") + + out, err := platformtest.Stdout(t, func() error { return gitops.DryRun(ctx, p.Client, gitops.Environment, []string{file}, false) }) + + require.NoError(t, err) + assert.Equal(t, file+" matches environment "+environmentID+".\n", out) + assert.Equal(t, []string{"Prod"}, p.Requests("GET /api/environments")[0].Query["search"]) +} + +func TestAnUnknownTeamWouldBeCreated(t *testing.T) { + p := platformtest.New(t) + p.Reply("GET /api/teams/NEW", platformtest.Reply{Status: http.StatusNotFound}) + file := write(t, "t.yml", "key: NEW\nname: New\nallowedActions: []\nallowedEnvironments: []\n") + + out, err := platformtest.Stdout(t, func() error { return gitops.DryRun(ctx, p.Client, gitops.Team, []string{file}, false) }) + + require.NoError(t, err) + assert.Equal(t, file+" would create a new team.\n", out) +} diff --git a/internal/hub/hub.go b/internal/hub/hub.go index 730acd5..f9f59e0 100644 --- a/internal/hub/hub.go +++ b/internal/hub/hub.go @@ -20,7 +20,7 @@ import ( // What the last synchronisation found and who edited the hub is the platform's. The // version is dropped as `service get` drops it. -var readOnly = []string{"version", "templates", "lastSync", "lastRepositoryChange", "syncError", "created", "createdBy", "edited", "editedBy"} +var ReadOnly = []string{"version", "templates", "lastSync", "lastRepositoryChange", "syncError", "created", "createdBy", "edited", "editedBy"} func uuid(id string) (openapi_types.UUID, error) { u, ok := resource.UUID(id) @@ -77,7 +77,7 @@ func Get(ctx context.Context, c *platform.Client, o GetOptions) error { if err != nil { return notFoundOr(err, o.ID, "Failed to get hub %s") } - if err := resource.Output(resource.Strip(doc, readOnly...), o.File, o.Type); err != nil { + if err := resource.Output(resource.Strip(doc, ReadOnly...), o.File, o.Type); err != nil { return err } if o.File != "" { @@ -107,7 +107,7 @@ func Apply(ctx context.Context, c *platform.Client, o ApplyOptions) error { Templates []any `json:"templates"` SyncError string `json:"syncError"` } - resp, err := c.UpsertHubWithBody(ctx, &api.UpsertHubParams{Synchronize: &o.Synchronize}, "application/json", resource.Body(resource.Strip(doc, readOnly...).Value())) + resp, err := c.UpsertHubWithBody(ctx, &api.UpsertHubParams{Synchronize: &o.Synchronize}, "application/json", resource.Body(resource.Strip(doc, ReadOnly...).Value())) resp, err = platform.Decode(resp, err, &saved) if err != nil { return resource.Applied{}, platform.Failed(err, "Failed to save hub %s", name) diff --git a/internal/integration/integration.go b/internal/integration/integration.go index 0dc0067..f25d480 100644 --- a/internal/integration/integration.go +++ b/internal/integration/integration.go @@ -13,6 +13,7 @@ import ( "strings" openapi_types "github.com/oapi-codegen/runtime/types" + "github.com/steadybit/cli/v6/internal/jsyaml" "github.com/steadybit/cli/v6/internal/output" "github.com/steadybit/cli/v6/internal/platform" "github.com/steadybit/cli/v6/internal/resource" @@ -21,7 +22,7 @@ import ( // The version is dropped as `service get` drops it: kept in a file, it turns every apply // after an edit in the UI into a conflict. -var readOnly = []string{"version"} +var ReadOnly = []string{"version"} type Kind struct { Name string // as the command names it @@ -29,6 +30,9 @@ type Kind struct { Plural string // The column that says where the integration reports to. Column, ColumnTitle string + // Secrets are the fields holding credentials; each value of a map among them is one. + // `export` masks them, so that a tenant kept in Git does not hold them. + Secrets []string list func(ctx context.Context, c *platform.Client) (*http.Response, error) get func(ctx context.Context, c *platform.Client, id openapi_types.UUID) (*http.Response, error) @@ -39,7 +43,9 @@ type Kind struct { var ( Webhook = Kind{ Name: "webhook", Title: "Webhook integration", Plural: "webhook integrations", Column: "url", ColumnTitle: "URL", - list: func(ctx context.Context, c *platform.Client) (*http.Response, error) { return c.GetCustomWebhooks(ctx) }, + // Headers carry API keys and tokens as often as anything else. + Secrets: []string{"secret", "headers"}, + list: func(ctx context.Context, c *platform.Client) (*http.Response, error) { return c.GetCustomWebhooks(ctx) }, get: func(ctx context.Context, c *platform.Client, id openapi_types.UUID) (*http.Response, error) { return c.GetCustomWebhook(ctx, id) }, @@ -52,6 +58,8 @@ var ( } Slack = Kind{ Name: "slack", Title: "Slack integration", Plural: "Slack integrations", Column: "channel", ColumnTitle: "Channel", + // The URL of a Slack incoming webhook is its credential: whoever has it can post. + Secrets: []string{"url"}, list: func(ctx context.Context, c *platform.Client) (*http.Response, error) { return c.GetSlackIntegrations(ctx) }, @@ -67,6 +75,7 @@ var ( } Preflight = Kind{ Name: "preflight", Title: "Preflight webhook", Plural: "preflight webhooks", Column: "url", ColumnTitle: "URL", + Secrets: []string{"secret", "headers"}, list: func(ctx context.Context, c *platform.Client) (*http.Response, error) { return c.GetPreflightWebhooks(ctx) }, @@ -98,6 +107,109 @@ var ( Kinds = []Kind{Webhook, Slack, Preflight, PreflightAction} ) +// Fetch gets one integration as `get` reads it. +func (k Kind) Fetch(ctx context.Context, c *platform.Client, id openapi_types.UUID) (*http.Response, error) { + return k.get(ctx, c, id) +} + +// FetchAll lists the integrations of the kind, as `list` reads them. +func (k Kind) FetchAll(ctx context.Context, c *platform.Client) (*http.Response, error) { + return k.list(ctx, c) +} + +// Mask is what `export` writes in place of a credential. Like the platform's own mask of +// a secret, it is only asterisks, which is how apply and diff tell one. +const Mask = "********" + +func Masked(v any) bool { + s, ok := v.(string) + return ok && s != "" && strings.Trim(s, "*") == "" +} + +// MaskSecrets replaces the credentials in an integration with the mask. +func (k Kind) MaskSecrets(doc *jsyaml.Map) { + for _, field := range k.Secrets { + switch v, _ := doc.Get(field); x := v.(type) { + case string: + if x != "" { + doc.Set(field, Mask) + } + case *jsyaml.Map: + for _, key := range x.Keys() { + if s, _ := x.Get(key); s != "" { + x.Set(key, Mask) + } + } + } + } +} + +// keepStored puts back what the platform holds in place of the masks `export` writes +// for the credentials it reads back in the clear, so that an exported file applies as +// it is. The secret it never reads back; a masked one is refused afterwards. +func (k Kind) keepStored(ctx context.Context, c *platform.Client, file string, doc *jsyaml.Map) error { + var stored *jsyaml.Map + loaded := false + value := func(field, key string) (any, error) { + if !loaded { + loaded = true + id, _ := doc.Get("id") + if u, ok := resource.UUID(fmt.Sprint(id)); ok { + d, _, err := platform.ReadDocument(k.get(ctx, c, u)) + if err != nil && !platform.IsStatus(err, http.StatusNotFound) { + return nil, platform.Failed(err, "Failed to get %s %s", k.Title, id) + } + if d != nil { + stored = d.Value() + } + } + } + name := field + var v any + if stored != nil { + v, _ = stored.Get(field) + if key != "" { + name = field + "." + key + m, _ := v.(*jsyaml.Map) + v = nil + if m != nil { + v, _ = m.Get(key) + } + } + } + if v == nil || v == "" || Masked(v) { + return nil, fmt.Errorf("%s file '%s' holds a masked %s, and the platform has none to keep. Put the value in.", k.Title, file, name) + } + return v, nil + } + for _, field := range k.Secrets { + if field == "secret" { + continue + } + switch v, _ := doc.Get(field); x := v.(type) { + case string: + if Masked(x) { + kept, err := value(field, "") + if err != nil { + return err + } + doc.Set(field, kept) + } + case *jsyaml.Map: + for _, key := range x.Keys() { + if s, _ := x.Get(key); Masked(s) { + kept, err := value(field, key) + if err != nil { + return err + } + x.Set(key, kept) + } + } + } + } + return nil +} + func (k Kind) uuid(id string) (openapi_types.UUID, error) { u, ok := resource.UUID(id) if !ok { @@ -157,7 +269,7 @@ func Get(ctx context.Context, c *platform.Client, k Kind, o GetOptions) error { if err != nil { return k.notFoundOr(err, o.ID, "Failed to get %s %s") } - if err := resource.Output(resource.Strip(doc, readOnly...), o.File, o.Type); err != nil { + if err := resource.Output(resource.Strip(doc, ReadOnly...), o.File, o.Type); err != nil { return err } if o.File != "" { @@ -177,13 +289,16 @@ func Apply(ctx context.Context, c *platform.Client, k Kind, o ApplyOptions) erro if name == "" { return resource.Applied{}, fmt.Errorf("%s file '%s' does not name the integration.", k.Title, file) } + if err := k.keepStored(ctx, c, file, doc.Value()); err != nil { + return resource.Applied{}, err + } // The platform masks secrets when reading them back and rejects the mask, while // leaving the secret out removes it. Neither is what a file from `get` means. - if secret, _ := doc.Get("secret"); secret != "" && strings.Trim(secret, "*") == "" { - return resource.Applied{}, fmt.Errorf("%s file '%s' holds the masked secret `get` writes. Put the secret in, or remove it for none.", k.Title, file) + if secret, _ := doc.Value().Get("secret"); Masked(secret) { + return resource.Applied{}, fmt.Errorf("%s file '%s' holds the masked secret `get` and `export` write. Put the secret in, or remove it for none.", k.Title, file) } var saved struct{ ID, Name string } - resp, err := k.upsert(ctx, c, resource.Body(resource.Strip(doc, readOnly...).Value())) + resp, err := k.upsert(ctx, c, resource.Body(resource.Strip(doc, ReadOnly...).Value())) resp, err = platform.Decode(resp, err, &saved) if err != nil { return resource.Applied{}, platform.Failed(err, "Failed to save %s %s", k.Title, name) diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go index 08a7c02..e0ff4b4 100644 --- a/internal/integration/integration_test.go +++ b/internal/integration/integration_test.go @@ -87,10 +87,36 @@ func TestApplyCreatesAndRefusesAMaskedSecret(t *testing.T) { content, _ := os.ReadFile(file) assert.Equal(t, "id: "+id+"\nname: Notify\nscope: GLOBAL\nurl: https://example.com\nsecret: s3cret\n", string(content)) err = integration.Apply(ctx, p.Client, integration.Webhook, integration.ApplyOptions{Files: []string{masked}}) - assert.EqualError(t, err, "Webhook integration file '"+masked+"' holds the masked secret `get` writes. Put the secret in, or remove it for none.") + assert.EqualError(t, err, "Webhook integration file '"+masked+"' holds the masked secret `get` and `export` write. Put the secret in, or remove it for none.") assert.Len(t, p.Requests("POST /api/integrations/webhook"), 1) } +// A header masked by `export` is sent as the platform holds it; one the platform does not +// have cannot be. +func TestApplyKeepsTheStoredValueOfAMaskedHeader(t *testing.T) { + p := platformtest.New(t) + p.Reply("GET /api/integrations/webhook/"+id, platformtest.Reply{Body: `{"id":"` + id + `","name":"Notify","headers":{"Authorization":"Bearer abc"}}`}) + p.Reply("POST /api/integrations/webhook", platformtest.Reply{JSON: map[string]any{"id": id, "name": "Notify"}}) + dir := t.TempDir() + file := filepath.Join(dir, "webhook.yml") + require.NoError(t, os.WriteFile(file, []byte("id: "+id+"\nname: Notify\nheaders:\n Authorization: '********'\n X-Team: '********'\n"), 0o644)) + + err := integration.Apply(ctx, p.Client, integration.Webhook, integration.ApplyOptions{Files: []string{file}}) + assert.EqualError(t, err, "Webhook integration file '"+file+"' holds a masked headers.X-Team, and the platform has none to keep. Put the value in.") + + require.NoError(t, os.WriteFile(file, []byte("id: "+id+"\nname: Notify\nheaders:\n Authorization: '********'\n X-Team: chaos\n"), 0o644)) + _, err = platformtest.Stdout(t, func() error { + return integration.Apply(ctx, p.Client, integration.Webhook, integration.ApplyOptions{Files: []string{file}}) + }) + + require.NoError(t, err) + sent := p.Requests("POST /api/integrations/webhook") + require.Len(t, sent, 1) + assert.Equal(t, map[string]any{"Authorization": "Bearer abc", "X-Team": "chaos"}, sent[0].JSON(t).(map[string]any)["headers"]) + content, _ := os.ReadFile(file) + assert.Contains(t, string(content), "Authorization: '********'", "the stored value is not written to the file") +} + func TestDelete(t *testing.T) { p := platformtest.New(t) p.Reply("DELETE /api/integrations/slack/"+id, platformtest.Reply{JSON: map[string]any{"id": id}}) diff --git a/internal/property/property.go b/internal/property/property.go index b234d78..1091ffd 100644 --- a/internal/property/property.go +++ b/internal/property/property.go @@ -21,7 +21,7 @@ import ( // The version is dropped as `service get` drops it: kept in a file, it turns every apply // after an edit in the UI into a conflict. -var readOnly = []string{"version"} +var ReadOnly = []string{"version"} func definitionNotFoundOr(err error, key, format string) error { if platform.IsStatus(err, http.StatusNotFound) { @@ -84,7 +84,7 @@ func GetDefinition(ctx context.Context, c *platform.Client, o GetDefinitionOptio if err != nil { return definitionNotFoundOr(err, o.Key, "Failed to get property definition %s") } - if err := resource.Output(resource.Strip(doc, readOnly...), o.File, o.Type); err != nil { + if err := resource.Output(resource.Strip(doc, ReadOnly...), o.File, o.Type); err != nil { return err } if o.File != "" { @@ -107,7 +107,7 @@ func ApplyDefinitions(ctx context.Context, c *platform.Client, o ApplyDefinition return resource.Applied{}, fmt.Errorf("Property definition file '%s' does not name the key.", file) } resp, err := c.UpsertPropertyDefinitionWithBody(ctx, &api.UpsertPropertyDefinitionParams{DeleteValues: &o.DeleteValues}, "application/json", - resource.Body(resource.Strip(doc, readOnly...).Value())) + resource.Body(resource.Strip(doc, ReadOnly...).Value())) _, resp, err = platform.Read(resp, err) if err != nil { return resource.Applied{}, platform.Failed(err, "Failed to save property definition %s", key) @@ -245,7 +245,7 @@ func GetAssociation(ctx context.Context, c *platform.Client, o GetAssociationOpt if err != nil { return associationNotFoundOr(err, o.ID, "Failed to get property association %s") } - if err := resource.Output(resource.Strip(doc, readOnly...), o.File, o.Type); err != nil { + if err := resource.Output(resource.Strip(doc, ReadOnly...), o.File, o.Type); err != nil { return err } if o.File != "" { @@ -266,7 +266,7 @@ func ApplyAssociations(ctx context.Context, c *platform.Client, o ApplyAssociati return resource.Applied{}, fmt.Errorf("Property association file '%s' does not name the property key.", file) } var saved struct{ ID, Key string } - resp, err := c.UpsertPropertyAssociationWithBody(ctx, "application/json", resource.Body(resource.Strip(doc, readOnly...).Value())) + resp, err := c.UpsertPropertyAssociationWithBody(ctx, "application/json", resource.Body(resource.Strip(doc, ReadOnly...).Value())) resp, err = platform.Decode(resp, err, &saved) if err != nil { return resource.Applied{}, platform.Failed(err, "Failed to save the property association of %s", key) diff --git a/internal/team/team.go b/internal/team/team.go index 5cb55c0..d5a2763 100644 --- a/internal/team/team.go +++ b/internal/team/team.go @@ -22,8 +22,8 @@ import ( // The version is dropped as `service get` drops it. Members are sent back as the platform // takes them, by username, email and role; the rest describes the user. var ( - readOnly = []string{"version"} - memberReadOnly = []string{"name", "pictureUrl", "managedBy"} + ReadOnly = []string{"version"} + MemberReadOnly = []string{"name", "pictureUrl", "managedBy"} ) func notFoundOr(err error, key, format string) error { @@ -85,12 +85,12 @@ func Get(ctx context.Context, c *platform.Client, o GetOptions) error { if err != nil { return notFoundOr(err, o.Key, "Failed to get team %s") } - resource.Strip(doc, readOnly...) + resource.Strip(doc, ReadOnly...) if members, ok := doc.Value().Get("members"); ok { list, _ := members.([]any) for _, m := range list { if member, ok := m.(*jsyaml.Map); ok { - for _, field := range memberReadOnly { + for _, field := range MemberReadOnly { member.Delete(field) } } @@ -117,7 +117,7 @@ func Apply(ctx context.Context, c *platform.Client, o ApplyOptions) error { if key == "" { return resource.Applied{}, fmt.Errorf("Team file '%s' does not name the team key.", file) } - resp, err := c.UpsertTeamWithBody(ctx, &api.UpsertTeamParams{}, "application/json", resource.Body(resource.Strip(doc, readOnly...).Value())) + resp, err := c.UpsertTeamWithBody(ctx, &api.UpsertTeamParams{}, "application/json", resource.Body(resource.Strip(doc, ReadOnly...).Value())) _, resp, err = platform.Read(resp, err) if err != nil { return resource.Applied{}, platform.Failed(err, "Failed to save team %s", key) diff --git a/internal/template/template.go b/internal/template/template.go index 1a0ea10..6f0314c 100644 --- a/internal/template/template.go +++ b/internal/template/template.go @@ -23,7 +23,7 @@ import ( // Who created and edited a template cannot be sent back. The version is dropped as // `service get` drops it, so that an edit in the UI does not turn the next apply into a // conflict. -var readOnly = []string{"created", "createdBy", "edited", "editedBy", "version"} +var ReadOnly = []string{"created", "createdBy", "edited", "editedBy", "version"} func notFoundOr(err error, id, format string) error { if platform.IsStatus(err, http.StatusNotFound) { @@ -118,7 +118,7 @@ func Get(ctx context.Context, c *platform.Client, o GetOptions) error { } doc = output.NewDocument(values) } - if err := resource.Output(resource.Strip(doc, readOnly...), o.File, o.Type); err != nil { + if err := resource.Output(resource.Strip(doc, ReadOnly...), o.File, o.Type); err != nil { return err } if o.File != "" { @@ -139,7 +139,7 @@ func Apply(ctx context.Context, c *platform.Client, o ApplyOptions) error { return resource.Applied{}, fmt.Errorf("Template file '%s' does not name a templateTitle.", file) } var saved struct{ ID, TemplateTitle string } - resp, err := c.UpsertExperimentTemplateWithBody(ctx, "application/json", resource.Body(resource.Strip(doc, readOnly...).Value())) + resp, err := c.UpsertExperimentTemplateWithBody(ctx, "application/json", resource.Body(resource.Strip(doc, ReadOnly...).Value())) resp, err = platform.Decode(resp, err, &saved) if err != nil { return resource.Applied{}, platform.Failed(err, "Failed to save experiment template %s", title) From 78c45177d9ff0bc8f7485cd1a8625308cd1a6fcd Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Tue, 29 Sep 2026 14:15:46 +0200 Subject: [PATCH 2/8] fix: export every custom template, and report an id that is no UUID The platform lists hidden templates, and those whose actions, target types or property definitions are not available right now, only when asked, so the tenant export silently left them out and the Git copy was incomplete. It now asks for both. An id that is no UUID was sent as the zero UUID. The export now fails on one, and a hub, template, environment or integration file holding one is reported with its name instead of being taken for a new one. --- internal/gitops/kinds.go | 8 ++++++-- internal/gitops/tenant.go | 11 +++++++++-- internal/gitops/tenant_test.go | 17 +++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/internal/gitops/kinds.go b/internal/gitops/kinds.go index 1e9c252..1f8e005 100644 --- a/internal/gitops/kinds.go +++ b/internal/gitops/kinds.go @@ -168,13 +168,17 @@ var ServiceProfile = Kind{ }, } -// byID finds the platform's version of a file by the id in it. +// byID finds the platform's version of a file by the id in it. A file without one is new; +// one whose id is no UUID names nothing on the platform, and applying it would not work. func byID(get func(ctx context.Context, c *platform.Client, id openapi_types.UUID) (*http.Response, error)) func(context.Context, *platform.Client, *jsyaml.Map) (string, *jsyaml.Map, error) { return func(ctx context.Context, c *platform.Client, local *jsyaml.Map) (string, *jsyaml.Map, error) { id := str(local, "id") + if id == "" { + return "", nil, nil + } u, ok := uuid(id) if !ok { - return "", nil, nil + return "", nil, fmt.Errorf("%s is not a valid id", id) } remote, err := fetch(get(ctx, c, u)) return id, remote, err diff --git a/internal/gitops/tenant.go b/internal/gitops/tenant.go index d15b625..0c8b923 100644 --- a/internal/gitops/tenant.go +++ b/internal/gitops/tenant.go @@ -5,6 +5,7 @@ package gitops import ( "context" + "errors" "fmt" "net/http" "path/filepath" @@ -50,7 +51,10 @@ func (e *tenantExport) each(ids []string, what string, get func(id string) (*htt func byUUID(get func(openapi_types.UUID) (*http.Response, error)) func(string) (*http.Response, error) { return func(id string) (*http.Response, error) { - u, _ := uuid(id) + u, ok := uuid(id) + if !ok { + return nil, errors.New("not a valid id") + } return get(u) } } @@ -152,7 +156,10 @@ func exportTenant(ctx context.Context, c *platform.Client, o ExportOptions) erro ID string `json:"id"` } `json:"templates"` } - resp, err = c.GetExperimentTemplates(ctx, &api.GetExperimentTemplatesParams{}) + // Hidden templates, and those whose actions, target types or property definitions are + // not available right now, are the tenant's too; the platform lists them only if asked. + all := true + resp, err = c.GetExperimentTemplates(ctx, &api.GetExperimentTemplatesParams{IncludeHidden: &all, IncludeNonAvailable: &all}) if _, err := platform.Decode(resp, err, &templates); err != nil { return platform.Failed(err, "Failed to get the experiment templates") } diff --git a/internal/gitops/tenant_test.go b/internal/gitops/tenant_test.go index 0a12479..36208f7 100644 --- a/internal/gitops/tenant_test.go +++ b/internal/gitops/tenant_test.go @@ -90,6 +90,9 @@ func TestExportTenantLeavesOutWhatThePlatformProvides(t *testing.T) { assert.NoFileExists(t, filepath.Join(dir, filepath.FromSlash(gone))) } assert.Empty(t, p.Requests("GET /api/experiments/templates/"+importedID), "a template imported from a hub is not even fetched") + listed := p.Requests("GET /api/experiments/templates")[0].Query + assert.Equal(t, []string{"true"}, listed["includeHidden"], "hidden templates are exported too") + assert.Equal(t, []string{"true"}, listed["includeNonAvailable"], "so are those whose actions are not available right now") out, err = platformtest.Stdout(t, func() error { return gitops.DiffProject(ctx, p.Client, dir) }) require.NoError(t, err) @@ -215,3 +218,17 @@ func TestAnUnknownTeamWouldBeCreated(t *testing.T) { require.NoError(t, err) assert.Equal(t, file+" would create a new team.\n", out) } + +// An id that is no UUID is reported, not sent to the platform as the zero UUID. +func TestAnInvalidIDIsReported(t *testing.T) { + p := fakeTenant(t) + p.Reply("GET /api/hubs", platformtest.Reply{JSON: map[string]any{"hubs": []any{map[string]any{"id": "not-an-id"}}}}) + + _, err := platformtest.Stdout(t, func() error { return gitops.Export(ctx, p.Client, gitops.ExportOptions{Directory: t.TempDir(), Tenant: true}) }) + + assert.EqualError(t, err, "Failed to get hub not-an-id: not a valid id") + + file := write(t, "hub.yml", "hubName: Own Hub\nrepositoryUrl: https://example.com/index.json\nid: not-an-id\n") + _, err = platformtest.Stdout(t, func() error { return gitops.DiffFiles(ctx, p.Client, gitops.Hub, []string{file}, false) }) + assert.EqualError(t, err, "Failed to get the hub for "+file+": not-an-id is not a valid id") +} From c5701b304da2c3985f15fcfa5002cc766eb54601 Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Tue, 29 Sep 2026 14:16:28 +0200 Subject: [PATCH 3/8] fix: never print a secret that is not a string Diffs masked only string values in secret fields, so a header value written as an unquoted YAML number was printed in the clear. Any non-empty value in a secret field is now masked, whatever its type. --- internal/gitops/kinds.go | 14 +++++++++----- internal/gitops/tenant_test.go | 12 ++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/internal/gitops/kinds.go b/internal/gitops/kinds.go index 1f8e005..8bbfbda 100644 --- a/internal/gitops/kinds.go +++ b/internal/gitops/kinds.go @@ -440,19 +440,23 @@ func hideSecrets(local, remote *jsyaml.Map, secrets []string) { } } +// hideSecret masks any value a secret field holds, whatever its type: an unquoted number +// in a file is as much a credential as a string. func hideSecret(local, remote *jsyaml.Map, key string) { lv, lok := local.Get(key) rv, rok := remote.Get(key) - rs, _ := rv.(string) - ls, _ := lv.(string) - if rok && rs != "" { + if rok && !blank(rv) { remote.Set(key, integration.Mask) } switch { - case !lok || ls == "" || ls == integration.Mask: - case integration.Masked(ls) || (rok && ls == rs): + case !lok || blank(lv) || lv == integration.Mask: + case integration.Masked(lv) || (rok && jsyaml.CompactJSON(lv) == jsyaml.CompactJSON(rv)): local.Set(key, integration.Mask) default: local.Set(key, integration.Mask+" (from the file)") } } + +func blank(v any) bool { + return v == nil || v == "" +} diff --git a/internal/gitops/tenant_test.go b/internal/gitops/tenant_test.go index 36208f7..439b1f5 100644 --- a/internal/gitops/tenant_test.go +++ b/internal/gitops/tenant_test.go @@ -168,6 +168,18 @@ func TestCredentialsAreNeverPrinted(t *testing.T) { assert.Contains(t, out, "- Authorization: '********'\n+ Authorization: '******** (from the file)'\n") assert.NotContains(t, out, "s3cret") assert.NotContains(t, out, "Bearer") + + // An unquoted number in the file is a credential all the same. + file = write(t, "n.yml", "id: "+webhookID+"\nscope: GLOBAL\nname: Notify\nurl: https://example.com/hook\nsecret: '********'\nevents:\n - '*'\n"+ + "targetAttributeIncludes:\n - '*'\nheaders:\n Authorization: Bearer abc\n X-Api-Key: 8675309\n") + + out, err = platformtest.Stdout(t, func() error { + return gitops.DiffFiles(ctx, p.Client, gitops.Integrations["webhook"], []string{file}, false) + }) + + assert.ErrorIs(t, err, gitops.ErrDifferent) + assert.Contains(t, out, "+ X-Api-Key: '******** (from the file)'\n") + assert.NotContains(t, out, "8675309") } // A team sent no actions may still wait and validate services, and a webhook sent no From 81a5bc1df54242e484008480f790c93699cd1e97 Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Tue, 29 Sep 2026 14:17:54 +0200 Subject: [PATCH 4/8] fix: match a team kept in Git by its key, not its id Teams were looked up by key but compared on their id, so an export diffed or applied on another platform, where the team has another id, showed an id difference forever and sent that platform a foreign id. The platform upserts a team by its key and ignores the id: on dev, a file with a foreign id and an existing key updated that team, and one with the team's id and a new key created a second team. The id is therefore neither exported nor compared, and `team apply` no longer sends it, so it cannot contradict the key should the platform ever start reading it. --- CHANGELOG.md | 3 ++- internal/gitops/kinds.go | 9 +++++---- internal/gitops/tenant_test.go | 22 +++++++++++++++++++++- internal/team/team.go | 4 +++- internal/team/team_test.go | 2 ++ 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ac3ac4..70b6056 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ environments, teams, hubs, templates, integrations and profiles in that order, before services, experiments and schedules. Hubs the platform connects itself, templates imported from a hub and Steadybit's service profiles are left out, as every platform - has them. + has them. Teams are kept without their id, which differs between platforms, and + matched by their key; `team apply` no longer sends the id, which the platform ignores. - Credentials of integrations are masked in exported files, and never printed by `diff`. A masked value matches whatever the platform holds, so that an exported tenant shows no drift and applies again: integrations that match are not applied, as the platform keeps diff --git a/internal/gitops/kinds.go b/internal/gitops/kinds.go index 8bbfbda..c4c18ca 100644 --- a/internal/gitops/kinds.go +++ b/internal/gitops/kinds.go @@ -230,14 +230,15 @@ var Environment = Kind{ }, } -// Team is matched by its key; the id `get` writes is not needed, so a file without one -// is not a difference. +// Team is matched by its key, as the platform upserts it. The id differs from one platform +// to the next, so it is neither exported nor compared: a team kept in Git matches its +// namesake anywhere. var Team = Kind{ Name: "team", - ReadOnly: append(append([]string{}, team.ReadOnly...), prefixed("members.", team.MemberReadOnly)...), + ReadOnly: append(append([]string{"id"}, team.ReadOnly...), prefixed("members.", team.MemberReadOnly)...), // Sent none, a team may still wait and validate services. Defaults: map[string]any{"allowedActions": []any{"wait", "service-validation"}, "managedBy": "MANUAL"}, - Identity: "id", + Identity: "key", Remote: func(ctx context.Context, c *platform.Client, local *jsyaml.Map) (string, *jsyaml.Map, error) { key := str(local, "key") if key == "" { diff --git a/internal/gitops/tenant_test.go b/internal/gitops/tenant_test.go index 439b1f5..ede6b1b 100644 --- a/internal/gitops/tenant_test.go +++ b/internal/gitops/tenant_test.go @@ -71,7 +71,7 @@ func TestExportTenantLeavesOutWhatThePlatformProvides(t *testing.T) { for file, content := range map[string]string{ "property-definitions/tribe.yaml": "key: tribe\nlabel: Tribe\ndataType: STRING\n", "environments/prod.yaml": "id: " + environmentID + "\nname: Prod\npredicate:\n operator: AND\n predicates: []\n", - "teams/adm.yaml": "id: " + teamID + "\nkey: ADM\nname: Admins\nallowedActions:\n - wait\n - service-validation\nallowedEnvironments:\n - Prod\nmanagedBy: MANUAL\n" + + "teams/adm.yaml": "key: ADM\nname: Admins\nallowedActions:\n - wait\n - service-validation\nallowedEnvironments:\n - Prod\nmanagedBy: MANUAL\n" + "members:\n - username: u1\n email: jane@example.com\n role: OWNER\n", "hubs/own-hub.yaml": "hubName: Own Hub\nrepositoryUrl: https://example.com/index.json\nid: " + hubID + "\n", "templates/pod-crash.yaml": "id: " + templateID + "\ntemplateTitle: Pod Crash\ntemplateDescription: d\nplaceholders: []\ntags: []\nlanes:\n - steps:\n - type: wait\n ignoreFailure: false\n" + @@ -204,6 +204,26 @@ func TestAHandWrittenTeamAndWebhookMatchRightAfterApply(t *testing.T) { assert.Contains(t, out, "-allowedActions:\n- - wait\n+allowedActions: []\n") } +// On another platform the team has another id. It is the same team by its key, and +// applying the file sends no id to contradict it. +func TestATeamMatchesByKeyOnAnotherPlatform(t *testing.T) { + p := platformtest.New(t) + p.Reply("GET /api/teams/ADM", platformtest.Reply{Body: `{"id":"` + environmentID + `","key":"ADM","name":"Admins","allowedActions":["wait","service-validation"],"allowedEnvironments":["Prod"],"managedBy":"MANUAL","version":0,"members":[]}`}) + p.Reply("POST /api/teams", platformtest.Reply{JSON: map[string]any{}}) + dir := t.TempDir() + file := filepath.Join(dir, "teams", "adm.yaml") + require.NoError(t, os.MkdirAll(filepath.Dir(file), 0o755)) + require.NoError(t, os.WriteFile(file, []byte("id: "+teamID+"\nkey: ADM\nname: Admins\nallowedActions:\n - wait\n - service-validation\nallowedEnvironments:\n - Prod\nmembers: []\n"), 0o644)) + + out, err := platformtest.Stdout(t, func() error { return gitops.DryRun(ctx, p.Client, gitops.Team, []string{file}, false) }) + require.NoError(t, err) + assert.Equal(t, file+" matches team ADM.\n", out) + + _, err = platformtest.Stdout(t, func() error { return gitops.ApplyProject(ctx, p.Client, gitops.ApplyOptions{Directory: dir}) }) + require.NoError(t, err) + assert.NotContains(t, p.Requests("POST /api/teams")[0].JSON(t), "id") +} + func TestEnvironmentsAreFoundByName(t *testing.T) { p := platformtest.New(t) p.Reply("GET /api/environments", platformtest.Reply{JSON: map[string]any{"environments": []any{ diff --git a/internal/team/team.go b/internal/team/team.go index d5a2763..86a293f 100644 --- a/internal/team/team.go +++ b/internal/team/team.go @@ -111,13 +111,15 @@ type ApplyOptions struct { } // Apply upserts teams by their key, which is what names a team; there is no id to write back. +// The id `get` writes is not sent: the platform goes by the key alone, and another +// platform's id in the file would only contradict it. func Apply(ctx context.Context, c *platform.Client, o ApplyOptions) error { return resource.ApplyFiles(o.Files, o.Recursive, "team", func(file string, doc *output.Document) (resource.Applied, error) { key, _ := doc.Get("key") if key == "" { return resource.Applied{}, fmt.Errorf("Team file '%s' does not name the team key.", file) } - resp, err := c.UpsertTeamWithBody(ctx, &api.UpsertTeamParams{}, "application/json", resource.Body(resource.Strip(doc, ReadOnly...).Value())) + resp, err := c.UpsertTeamWithBody(ctx, &api.UpsertTeamParams{}, "application/json", resource.Body(resource.Strip(doc, append([]string{"id"}, ReadOnly...)...).Value())) _, resp, err = platform.Read(resp, err) if err != nil { return resource.Applied{}, platform.Failed(err, "Failed to save team %s", key) diff --git a/internal/team/team_test.go b/internal/team/team_test.go index 78d6dac..cfe3016 100644 --- a/internal/team/team_test.go +++ b/internal/team/team_test.go @@ -55,6 +55,8 @@ func TestGetAndApplyRoundTripByKey(t *testing.T) { sent := p.Requests("POST /api/teams")[0].JSON(t).(map[string]any) assert.Equal(t, []any{map[string]any{"username": "u-1", "email": "jane@example.com", "role": "OWNER"}}, sent["members"]) assert.Equal(t, "MANUAL", sent["managedBy"]) + assert.Contains(t, string(before), "id: 0190d7b2-0000-7000-8000-000000000001") + assert.NotContains(t, sent, "id", "the platform goes by the key") } func TestApplyWritesNothingBack(t *testing.T) { From fdc85e0558d35e7126426b3f50b91eb7a9ad7083 Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Tue, 29 Sep 2026 14:19:00 +0200 Subject: [PATCH 5/8] fix: synchronize hubs when applying a project Hubs were saved without being synchronized, so a restored tenant had the hub but none of its templates by the time the service profiles were applied, and profiles naming those templates failed. `apply -d` now synchronizes each hub as `hub apply --synchronize` does, and a hub that cannot be synchronized ends the apply with the platform's reason, before anything that depends on it. --- CHANGELOG.md | 10 ++++++---- README.md | 3 ++- internal/gitops/project.go | 5 ++++- internal/gitops/tenant_test.go | 22 +++++++++++++++++++++- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70b6056..7743a03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,12 @@ integrations and custom service profiles. `diff -d dir` and `apply -d dir [--dry-run]` take such a directory as they take a team's, and apply property definitions, environments, teams, hubs, templates, integrations and profiles in that order, before - services, experiments and schedules. Hubs the platform connects itself, templates - imported from a hub and Steadybit's service profiles are left out, as every platform - has them. Teams are kept without their id, which differs between platforms, and - matched by their key; `team apply` no longer sends the id, which the platform ignores. + services, experiments and schedules. Hubs are synchronized as they are applied, so + that the service profiles find the templates they name. Hubs the platform connects + itself, templates imported from a hub and Steadybit's service profiles are left out, + as every platform has them. Teams are kept without their id, which differs between + platforms, and matched by their key; `team apply` no longer sends the id, which the + platform ignores. - Credentials of integrations are masked in exported files, and never printed by `diff`. A masked value matches whatever the platform holds, so that an exported tenant shows no drift and applies again: integrations that match are not applied, as the platform keeps diff --git a/README.md b/README.md index eaab9e1..c4ae67a 100644 --- a/README.md +++ b/README.md @@ -278,7 +278,8 @@ steadybit apply -d ./platform # definitions, environments, teams, hu ``` What the platform provides is left out: the hubs it connects, the templates imported from -a hub (`template import` brings them back) and Steadybit's service profiles. +a hub (`template import` brings them back) and Steadybit's service profiles. Hubs are +synchronized as they are applied, so that the service profiles find their templates. Credentials of integrations (secrets, header values, Slack webhook URLs) are written as `'********'`. A mask stands for what the platform holds: `diff` does not report it, and diff --git a/internal/gitops/project.go b/internal/gitops/project.go index c32ca8e..7261845 100644 --- a/internal/gitops/project.go +++ b/internal/gitops/project.go @@ -49,8 +49,11 @@ var projectKinds = []struct { {"teams", Team, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { return team.Apply(ctx, c, team.ApplyOptions{Files: []string{path}, Recursive: true}) }}, + // Synchronized, as `hub apply --synchronize` does: service profiles name the templates + // a hub brings, so on a restored tenant those have to be there first. A hub that cannot + // be synchronized ends the apply. {"hubs", Hub, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { - return hub.Apply(ctx, c, hub.ApplyOptions{Files: []string{path}, Recursive: true}) + return hub.Apply(ctx, c, hub.ApplyOptions{Files: []string{path}, Recursive: true, Synchronize: true}) }}, {"templates", Template, func(ctx context.Context, c *platform.Client, path string, _ ApplyOptions) error { return template.Apply(ctx, c, template.ApplyOptions{Files: []string{path}, Recursive: true}) diff --git a/internal/gitops/tenant_test.go b/internal/gitops/tenant_test.go index ede6b1b..43f49cb 100644 --- a/internal/gitops/tenant_test.go +++ b/internal/gitops/tenant_test.go @@ -117,7 +117,8 @@ func TestApplyingAnExportedTenantSkipsMatchingIntegrations(t *testing.T) { assert.Contains(t, out, "chat.yaml matches Slack integration "+slackID+".\n") assert.Empty(t, p.Requests("POST /api/integrations/webhook")) assert.Empty(t, p.Requests("POST /api/integrations/slack")) - order := []string{"Property definition tribe", "Environment", "Team ADM", "Hub", "Experiment template", "Service profile"} + assert.Equal(t, []string{"true"}, p.Requests("POST /api/hubs")[0].Query["synchronize"], "the hub's templates are there for the service profiles") + order :=[]string{"Property definition tribe", "Environment", "Team ADM", "Hub", "Experiment template", "Service profile"} last := -1 for _, line := range order { i := strings.Index(out, line) @@ -126,6 +127,25 @@ func TestApplyingAnExportedTenantSkipsMatchingIntegrations(t *testing.T) { } } +// Service profiles may name the templates a hub brings, so a hub that cannot be +// synchronized ends the apply before them. +func TestAHubThatCannotBeSynchronizedEndsTheApply(t *testing.T) { + p := fakeTenant(t) + dir := t.TempDir() + _, err := platformtest.Stdout(t, func() error { return gitops.Export(ctx, p.Client, gitops.ExportOptions{Directory: dir, Tenant: true}) }) + require.NoError(t, err) + for _, route := range []string{"POST /api/properties/definitions", "POST /api/environments", "POST /api/teams", "POST /api/experiments/templates", "POST /api/services/profiles"} { + p.Reply(route, platformtest.Reply{JSON: map[string]any{}}) + } + p.Reply("POST /api/hubs", platformtest.Reply{JSON: map[string]any{"id": hubID, "hubName": "Own Hub", "syncError": "index.json not found"}}) + + _, err = platformtest.Stdout(t, func() error { return gitops.ApplyProject(ctx, p.Client, gitops.ApplyOptions{Directory: dir}) }) + + assert.EqualError(t, err, "Hub Own Hub could not be synchronized: index.json not found") + assert.Empty(t, p.Requests("POST /api/experiments/templates")) + assert.Empty(t, p.Requests("POST /api/services/profiles")) +} + // A changed integration is applied with the credentials the platform reads back in the // clear; its secret it never does, so that has to be put in. func TestAChangedExportedIntegrationKeepsItsCredentials(t *testing.T) { From be35806871a56ecd8faebada2e5ad3a14cebe093 Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Tue, 29 Sep 2026 14:20:15 +0200 Subject: [PATCH 6/8] fix: let integration apply skip an exported file left as it is `integration apply` refused an untouched exported file for its masked secret, while `apply -d` skipped the same file as matching the platform. A file holding a masked secret that otherwise matches what the platform holds now has nothing to apply and is reported as unchanged; a changed one still needs the secret put in. The platform's version is read once for both the masked header values and this check. --- CHANGELOG.md | 3 +- internal/integration/integration.go | 56 +++++++++++++++++++++--- internal/integration/integration_test.go | 24 ++++++++++ 3 files changed, 75 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7743a03..88f5c12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,8 @@ A masked value matches whatever the platform holds, so that an exported tenant shows no drift and applies again: integrations that match are not applied, as the platform keeps no secret it is not sent, and `integration ... apply` sends the stored header values and - Slack URLs in place of their masks. + Slack URLs in place of their masks, and leaves out a file with a masked secret that + matches the platform. - Experiment templates, environments, teams, property definitions, hubs and integrations have a `diff`, and their `apply` a `--dry-run`. The actions a team is given when sent none, and the target attributes a webhook reports when sent none, are not differences. diff --git a/internal/integration/integration.go b/internal/integration/integration.go index f25d480..1ad1c7c 100644 --- a/internal/integration/integration.go +++ b/internal/integration/integration.go @@ -144,13 +144,12 @@ func (k Kind) MaskSecrets(doc *jsyaml.Map) { } } -// keepStored puts back what the platform holds in place of the masks `export` writes -// for the credentials it reads back in the clear, so that an exported file applies as -// it is. The secret it never reads back; a masked one is refused afterwards. -func (k Kind) keepStored(ctx context.Context, c *platform.Client, file string, doc *jsyaml.Map) error { +// stored reads, once and only when asked, what the platform holds for the integration +// the file names: nil when it names none. +func (k Kind) stored(ctx context.Context, c *platform.Client, doc *jsyaml.Map) func() (*jsyaml.Map, error) { var stored *jsyaml.Map loaded := false - value := func(field, key string) (any, error) { + return func() (*jsyaml.Map, error) { if !loaded { loaded = true id, _ := doc.Get("id") @@ -164,6 +163,38 @@ func (k Kind) keepStored(ctx context.Context, c *platform.Client, file string, d } } } + return stored, nil + } +} + +// unchanged tells a file that holds what the platform does, its secret aside, which a +// mask in the file stands for. +func unchanged(doc, stored *jsyaml.Map) bool { + if stored == nil { + return false + } + if secret, _ := stored.Get("secret"); secret == nil || secret == "" { + return false + } + without := func(m *jsyaml.Map) string { + c := jsyaml.Clone(m).(*jsyaml.Map) + for _, f := range append([]string{"secret"}, ReadOnly...) { + c.Delete(f) + } + return jsyaml.CompactJSON(c) + } + return without(doc) == without(stored) +} + +// keepStored puts back what the platform holds in place of the masks `export` writes +// for the credentials it reads back in the clear, so that an exported file applies as +// it is. The secret it never reads back; a masked one is refused afterwards. +func (k Kind) keepStored(file string, doc *jsyaml.Map, load func() (*jsyaml.Map, error)) error { + value := func(field, key string) (any, error) { + stored, err := load() + if err != nil { + return nil, err + } name := field var v any if stored != nil { @@ -289,12 +320,23 @@ func Apply(ctx context.Context, c *platform.Client, k Kind, o ApplyOptions) erro if name == "" { return resource.Applied{}, fmt.Errorf("%s file '%s' does not name the integration.", k.Title, file) } - if err := k.keepStored(ctx, c, file, doc.Value()); err != nil { + stored := k.stored(ctx, c, doc.Value()) + if err := k.keepStored(file, doc.Value(), stored); err != nil { return resource.Applied{}, err } // The platform masks secrets when reading them back and rejects the mask, while - // leaving the secret out removes it. Neither is what a file from `get` means. + // leaving the secret out removes it. Neither is what a file from `get` means. Left + // as it was written, though, the file has nothing to apply, as `apply -d` finds. if secret, _ := doc.Value().Get("secret"); Masked(secret) { + held, err := stored() + if err != nil { + return resource.Applied{}, err + } + if unchanged(doc.Value(), held) { + id, _ := doc.Get("id") + fmt.Printf("%s %s (%s) unchanged.\n", k.Title, name, id) + return resource.Applied{}, nil + } return resource.Applied{}, fmt.Errorf("%s file '%s' holds the masked secret `get` and `export` write. Put the secret in, or remove it for none.", k.Title, file) } var saved struct{ ID, Name string } diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go index e0ff4b4..282d2c5 100644 --- a/internal/integration/integration_test.go +++ b/internal/integration/integration_test.go @@ -72,6 +72,7 @@ func TestGetAndApplyRoundTrip(t *testing.T) { func TestApplyCreatesAndRefusesAMaskedSecret(t *testing.T) { p := platformtest.New(t) p.Reply("POST /api/integrations/webhook", platformtest.Reply{Status: http.StatusCreated, JSON: map[string]any{"id": id, "name": "Notify"}}) + p.Reply("GET /api/integrations/webhook/"+id, platformtest.Reply{Status: http.StatusNotFound}) dir := t.TempDir() file := filepath.Join(dir, "webhook.yml") require.NoError(t, os.WriteFile(file, []byte("name: Notify\nscope: GLOBAL\nurl: https://example.com\nsecret: s3cret\n"), 0o644)) @@ -117,6 +118,29 @@ func TestApplyKeepsTheStoredValueOfAMaskedHeader(t *testing.T) { assert.Contains(t, string(content), "Authorization: '********'", "the stored value is not written to the file") } +// An exported file left as it is has nothing to apply, masked secret and all, as +// `apply -d` finds; changed, it needs the secret put in. +func TestApplySkipsAnUnchangedFileWithAMaskedSecret(t *testing.T) { + p := platformtest.New(t) + p.Reply("GET /api/integrations/webhook/"+id, platformtest.Reply{Body: `{"id":"` + id + `","version":2,"name":"Notify","url":"https://example.com","secret":"****************","headers":{"Authorization":"Bearer abc"}}`}) + file := filepath.Join(t.TempDir(), "webhook.yml") + require.NoError(t, os.WriteFile(file, []byte("id: "+id+"\nname: Notify\nurl: https://example.com\nsecret: '********'\nheaders:\n Authorization: '********'\n"), 0o644)) + + out, err := platformtest.Stdout(t, func() error { + return integration.Apply(ctx, p.Client, integration.Webhook, integration.ApplyOptions{Files: []string{file}}) + }) + + require.NoError(t, err) + assert.Equal(t, "Webhook integration Notify ("+id+") unchanged.\n", out) + assert.Empty(t, p.Requests("POST /api/integrations/webhook")) + assert.Len(t, p.Requests("GET /api/integrations/webhook/"+id), 1, "the platform's version is read once") + + require.NoError(t, os.WriteFile(file, []byte("id: "+id+"\nname: Notify\nurl: https://example.org\nsecret: '********'\nheaders:\n Authorization: '********'\n"), 0o644)) + err = integration.Apply(ctx, p.Client, integration.Webhook, integration.ApplyOptions{Files: []string{file}}) + assert.EqualError(t, err, "Webhook integration file '"+file+"' holds the masked secret `get` and `export` write. Put the secret in, or remove it for none.") + assert.Empty(t, p.Requests("POST /api/integrations/webhook")) +} + func TestDelete(t *testing.T) { p := platformtest.New(t) p.Reply("DELETE /api/integrations/slack/"+id, platformtest.Reply{JSON: map[string]any{"id": id}}) From ff429467b2ed0c52f93c5ed305eadc64bbc01c08 Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Tue, 29 Sep 2026 14:20:22 +0200 Subject: [PATCH 7/8] docs: title the changelog section v6.1.0, as the other open changes do --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88f5c12..84e263c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## v6.1.0 (unreleased) +## v6.1.0 - `export --tenant -d dir` writes the tenant's configuration, to keep in Git: experiment templates, environments, teams, property definitions, hubs, webhook, Slack and preflight From 70b8ed80a74f83d9a833edf20629fbc4b5c6ae00 Mon Sep 17 00:00:00 2001 From: "antoine.choimet" <12182686+achoimet@users.noreply.github.com.> Date: Tue, 29 Sep 2026 14:21:02 +0200 Subject: [PATCH 8/8] chore: gofmt the tenant tests --- internal/gitops/tenant_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/gitops/tenant_test.go b/internal/gitops/tenant_test.go index 43f49cb..20af628 100644 --- a/internal/gitops/tenant_test.go +++ b/internal/gitops/tenant_test.go @@ -118,7 +118,7 @@ func TestApplyingAnExportedTenantSkipsMatchingIntegrations(t *testing.T) { assert.Empty(t, p.Requests("POST /api/integrations/webhook")) assert.Empty(t, p.Requests("POST /api/integrations/slack")) assert.Equal(t, []string{"true"}, p.Requests("POST /api/hubs")[0].Query["synchronize"], "the hub's templates are there for the service profiles") - order :=[]string{"Property definition tribe", "Environment", "Team ADM", "Hub", "Experiment template", "Service profile"} + order := []string{"Property definition tribe", "Environment", "Team ADM", "Hub", "Experiment template", "Service profile"} last := -1 for _, line := range order { i := strings.Index(out, line) @@ -276,7 +276,9 @@ func TestAnInvalidIDIsReported(t *testing.T) { p := fakeTenant(t) p.Reply("GET /api/hubs", platformtest.Reply{JSON: map[string]any{"hubs": []any{map[string]any{"id": "not-an-id"}}}}) - _, err := platformtest.Stdout(t, func() error { return gitops.Export(ctx, p.Client, gitops.ExportOptions{Directory: t.TempDir(), Tenant: true}) }) + _, err := platformtest.Stdout(t, func() error { + return gitops.Export(ctx, p.Client, gitops.ExportOptions{Directory: t.TempDir(), Tenant: true}) + }) assert.EqualError(t, err, "Failed to get hub not-an-id: not a valid id")