From 213b9cdd957e6eed2cf1fce828de4f2803f90559 Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 15:19:00 +0200 Subject: [PATCH] feat(gcp): bootstrap two data centers with --multi-dc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires up multi-data-center support: --multi-dc bootstraps a second data center in the same project, sharing the VPC, jumpbox and PostgreSQL server but running its own Kubernetes and Ceph cluster. The secondary data center's config and vault are derived from the primary's after its secrets exist, which is why Bootstrap handles it in a loop after the primary's config is written: - the config is cloned and its data-center-scoped fields cleared, so the installer's dataCenters topology, the domains and the shared registry are inherited while the ingress CA and cephadm key are regenerated; - the vault is derived through DeriveDataCenterVault, keeping the postgres roles and token keys and dropping the per-cluster secrets; - postgres becomes mode: external pointing at the shared server's internal IP, which is what its certificate's only SAN carries, and "postgres" is appended to operations.skip in the config so manual re-runs on the jumpbox skip it too. A post-generation check fails the bootstrap if secrets that must match across data centers diverged, or if a per-cluster secret was inherited instead of regenerated — both would otherwise surface long after the fact, against a live shared database. Installs run strictly sequentially in ascending data center order, since the primary's install creates the database, roles and schema the others reuse. validateMultiDC rejects the combinations that cannot work (--write-config=false, an explicit --datacenter-id, an empty datacenter name), and the CLI prints one install command per data center in the required order. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- .gitignore | 4 + cli/cmd/bootstrap_gcp.go | 18 +- docs/oms_beta_bootstrap-gcp.md | 1 + internal/bootstrap/gcp/gcp.go | 80 +++++- internal/bootstrap/gcp/install_config.go | 268 ++++++++++++++++-- internal/bootstrap/gcp/install_config_test.go | 4 + 6 files changed, 344 insertions(+), 31 deletions(-) diff --git a/.gitignore b/.gitignore index eb2c4d4b..011d5c24 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,10 @@ internal/util/testdata/ config.yaml prod.vault.yaml configure-k0s.sh +# ... and their per-data-center variants written by a --multi-dc bootstrap +config-dc*.yaml +prod-dc*.vault.yaml +configure-k0s-dc*.sh # Debugger files __debug* diff --git a/cli/cmd/bootstrap_gcp.go b/cli/cmd/bootstrap_gcp.go index 8f87eae5..b9f557eb 100644 --- a/cli/cmd/bootstrap_gcp.go +++ b/cli/cmd/bootstrap_gcp.go @@ -89,6 +89,7 @@ func AddBootstrapGcpCmd(parent *cobra.Command, opts *util.GlobalOptions) { flags.BoolVar(&bootstrapGcpCmd.CodesphereEnv.SpotVMs, "spot-vms", false, "Use Spot VMs for Codesphere infrastructure. Falls back to standard VMs if spot capacity unavailable. Mutually exclusive with --preemptible (default: false)") flags.IntVar(&bootstrapGcpCmd.CodesphereEnv.DatacenterID, "datacenter-id", 1, "Datacenter ID (default: 1)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.DatacenterName, "datacenter-name", "dev", "Datacenter name (default: dev)") + flags.BoolVar(&bootstrapGcpCmd.CodesphereEnv.MultiDC, "multi-dc", false, "Bootstrap two data centers that share one PostgreSQL server but run separate Kubernetes and Ceph clusters. Doubles the Ceph and k0s nodes to 14 VMs (~100 vCPUs) and reserves 6 static IPs, so the region's CPU quota may need raising. Cannot be combined with --datacenter-id. (default: false)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.CustomPgIP, "custom-pg-ip", "", "Custom PostgreSQL IP (optional)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.Region, "region", "europe-west4", "GCP Region (default: europe-west4)") flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.Zone, "zone", "europe-west4-a", "GCP Zone (default: europe-west4-a)") @@ -174,6 +175,8 @@ func (c *BootstrapGcpCmd) BootstrapGcp() error { c.CodesphereEnv.RegistryType = gcp.RegistryType(c.InputRegistryType) c.CodesphereEnv.OmsWorkdir = c.Env.GetOmsWorkdir() + // The value alone cannot distinguish the default 1 from an explicit --datacenter-id=1. + c.CodesphereEnv.DatacenterIDExplicit = c.cmd.Flags().Changed("datacenter-id") if c.CodesphereEnv.GitHubPAT != "" { c.CodesphereEnv.RegistryType = gcp.RegistryTypeGitHub if c.CodesphereEnv.RegistryUser == "" { @@ -208,18 +211,27 @@ func (c *BootstrapGcpCmd) BootstrapGcp() error { if bs.Env.InstallVersion != "" { log.Printf("Access Codesphere in your web browser at https://cs.%s", bs.Env.BaseDomain) + for _, dc := range bs.Env.DataCenters { + log.Printf("Data center %d hosts workspaces under %s", dc.ID, dc.WorkspaceHostingBaseDomain) + } return nil } packageName := "-installer" - installCmd := "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml" if gcp.RegistryType(bs.Env.RegistryType) == gcp.RegistryTypeGitHub { log.Printf("You set a GitHub PAT for direct image access. Make sure to use a lite package, as VM root disk sizes are reduced.") - installCmd += " -s load-container-images" packageName += "-lite" } - log.Printf("example install command (run from jumpbox):\n%s -p %s.tar.gz", installCmd, packageName) + packageFile := packageName + ".tar.gz" + if len(bs.Env.DataCenters) > 1 { + log.Printf("example install commands (run from jumpbox). Run the data center 1 command to completion first — the other data centers share its database:") + } else { + log.Printf("example install command (run from jumpbox):") + } + for _, dc := range bs.Env.DataCenters { + log.Printf("# data center %d\n%s", dc.ID, bs.InstallCommand(dc, packageFile)) + } return nil } diff --git a/docs/oms_beta_bootstrap-gcp.md b/docs/oms_beta_bootstrap-gcp.md index b19a359f..eb3facf0 100644 --- a/docs/oms_beta_bootstrap-gcp.md +++ b/docs/oms_beta_bootstrap-gcp.md @@ -57,6 +57,7 @@ oms beta bootstrap-gcp [flags] --install-version string Codesphere version to install (default: none) --internal-flags stringArray Internal flags to enable in Codesphere installation (optional) (default [headless-services,vcluster,custom-service-image,ms-in-ls]) --local-trace-endpoint string Endpoint for exporting traces to an in-cluster storage (optional) + --multi-dc Bootstrap two data centers that share one PostgreSQL server but run separate Kubernetes and Ceph clusters. Doubles the Ceph and k0s nodes to 14 VMs (~100 vCPUs) and reserves 6 static IPs, so the region's CPU quota may need raising. Cannot be combined with --datacenter-id. (default: false) --oidc-client-id string OIDC OAuth provider Client ID (optional) --oidc-client-secret string OIDC OAuth provider Client Secret (optional) --oidc-issuer-url string OIDC OAuth provider issuer URL (optional) diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index b2e565bd..5da46e0d 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -151,6 +151,11 @@ func (b *GCPBootstrapper) primaryDC() *datacenter.DataCenter { return b.Env.DataCenters[0] } +// secondaryDCs returns every data center apart from the primary one. +func (b *GCPBootstrapper) secondaryDCs() []*datacenter.DataCenter { + return b.Env.DataCenters[1:] +} + // allNodes returns every node of the project: the jumpbox, the shared postgres node and all // data centers' Ceph and k0s nodes. func (b *GCPBootstrapper) allNodes() []*node.Node { @@ -419,19 +424,41 @@ func (b *GCPBootstrapper) Bootstrap() error { } if b.Env.WriteConfig { - err = b.stlog.Step("Update install config", b.UpdateInstallConfig) + err = b.writeDataCenterConfig(b.primaryDC()) if err != nil { - return fmt.Errorf("failed to update install config: %w", err) + return err + } + } + + // Secondary data centers fall back to deriving their config and vault from the primary one, + // so this has to run after the primary's secrets were generated above. + for _, dc := range b.secondaryDCs() { + err = b.stlog.Step(dc.StepName("Ensure install config"), func() error { + return b.ensureInstallConfig(dc) + }) + if err != nil { + return fmt.Errorf("failed to ensure install config of data center %d: %w", dc.ID, err) } - err = b.stlog.Step("Ensure age key", b.EnsureAgeKey) + err = b.stlog.Step(dc.StepName("Ensure secrets"), func() error { + return b.ensureSecrets(dc) + }) if err != nil { - return fmt.Errorf("failed to ensure age key: %w", err) + return fmt.Errorf("failed to ensure secrets of data center %d: %w", dc.ID, err) } - err = b.stlog.Step("Encrypt vault", b.EncryptVault) + err = b.stlog.Step(dc.StepName("Derive config and vault"), func() error { + return b.seedSecondaryDataCenter(b.primaryDC(), dc) + }) if err != nil { - return fmt.Errorf("failed to encrypt vault: %w", err) + return fmt.Errorf("failed to derive data center %d: %w", dc.ID, err) + } + + if b.Env.WriteConfig { + err = b.writeDataCenterConfig(dc) + if err != nil { + return err + } } } @@ -557,9 +584,50 @@ func (b *GCPBootstrapper) ValidateInput() error { return err } + err = b.validateMultiDC() + if err != nil { + return err + } + return b.validateTelemetryExportParams() } +// validateMultiDC rejects flag combinations a multi-data-center bootstrap cannot satisfy. +func (b *GCPBootstrapper) validateMultiDC() error { + if !b.Env.MultiDC { + return nil + } + + // A secondary data center's config and vault are derived from the primary's, which only + // happens when configs are written. + if !b.Env.WriteConfig { + return fmt.Errorf("multi-dc requires write-config to be enabled") + } + + // The data center IDs are derived (1 and 2) and drive the workspace hosting domains. + if b.Env.DatacenterIDExplicit { + return fmt.Errorf("datacenter-id cannot be combined with multi-dc, the IDs are derived") + } + + // The k0s cluster is named codesphere-, so both names must be set and + // distinct. BuildDataCenters derives the second one by suffixing the first. + if b.Env.DatacenterName == "" { + return fmt.Errorf("datacenter-name is required with multi-dc") + } + + // Every data center gets its own local config and vault, derived by suffixing these paths. + for name, path := range map[string]string{ + "install-config": b.Env.InstallConfigPath, + "secrets-file": b.Env.SecretsFilePath, + } { + if path == "" { + return fmt.Errorf("cannot derive a per-data-center path: %s is empty", name) + } + } + + return nil +} + func (b *GCPBootstrapper) validateClusterAdminEmail() error { if b.Env.ClusterAdminEmail == "" { return nil diff --git a/internal/bootstrap/gcp/install_config.go b/internal/bootstrap/gcp/install_config.go index 5e619145..0e2ae2a3 100644 --- a/internal/bootstrap/gcp/install_config.go +++ b/internal/bootstrap/gcp/install_config.go @@ -4,10 +4,14 @@ package gcp import ( + "errors" "fmt" + "reflect" + "slices" "github.com/codesphere-cloud/oms/internal/bootstrap" "github.com/codesphere-cloud/oms/internal/bootstrap/datacenter" + "github.com/codesphere-cloud/oms/internal/codesphere" "github.com/codesphere-cloud/oms/internal/installer/files" "github.com/codesphere-cloud/oms/internal/installer/secrets" "github.com/codesphere-cloud/oms/internal/util" @@ -15,9 +19,17 @@ import ( const ( remoteInstallConfigPath string = "/etc/codesphere/config.yaml" + // sharedPostgresPort is the port the shared PostgreSQL server listens on. Secondary data + // centers need it spelled out because they connect to it as an external server. + sharedPostgresPort int = 5432 ) -// EnsureInstallConfig prepares the primary data center's install config. +// errRemoteConfigMissing reports that the jumpbox holds no config for a data center. Recovering +// a secondary data center tolerates this, since --multi-dc can add one to an existing project. +var errRemoteConfigMissing = errors.New("no install config found on the jumpbox") + +// EnsureInstallConfig prepares the primary data center's install config. Secondary data centers +// are handled separately in Bootstrap, after the primary's secrets exist. func (b *GCPBootstrapper) EnsureInstallConfig() error { b.ensureDataCenters() @@ -30,7 +42,12 @@ func (b *GCPBootstrapper) ensureInstallConfig(dc *datacenter.DataCenter) error { // recovery will overwrite local config or create a new file if b.Env.RecoverConfig { err := b.recoverConfig(dc) - if err != nil { + if errors.Is(err, errRemoteConfigMissing) && !dc.IsPrimary() { + // A secondary data center may not exist on the jumpbox yet, which is the case when + // --multi-dc is used to add one to an existing single-DC project. Its config is + // derived from the primary's instead. + b.stlog.Logf("No config found on the jumpbox for data center %d, generating a new one", dc.ID) + } else if err != nil { return fmt.Errorf("failed to recover config: %w", err) } } @@ -46,14 +63,17 @@ func (b *GCPBootstrapper) ensureInstallConfig(dc *datacenter.DataCenter) error { } dc.ExistingConfigUsed = true - } else { + dc.InstallConfig = dc.ConfigManager.GetInstallConfig() + } else if dc.IsPrimary() { err := dc.ConfigManager.ApplyProfile("minimal") if err != nil { return fmt.Errorf("failed to apply profile: %w", err) } + dc.InstallConfig = dc.ConfigManager.GetInstallConfig() } + // A secondary data center without a config of its own is left unset here, so + // seedSecondaryDataCenter can derive it from the primary data center instead of the profile. - dc.InstallConfig = dc.ConfigManager.GetInstallConfig() b.mirrorPrimaryDataCenter() return nil @@ -88,6 +108,12 @@ func (b *GCPBootstrapper) recoverConfig(dc *datacenter.DataCenter) error { } b.Env.Jumpbox = jumpbox + // Only a secondary data center may legitimately be absent from the jumpbox, so only there is + // it worth probing first; the primary's missing config stays a download failure. + if !dc.IsPrimary() && !b.Env.Jumpbox.NodeClient.HasFile(jumpbox, dc.RemoteConfigPath) { + return fmt.Errorf("%w at %s", errRemoteConfigMissing, dc.RemoteConfigPath) + } + err = b.Env.Jumpbox.NodeClient.DownloadFile(jumpbox, dc.RemoteConfigPath, dc.InstallConfigPath) if err != nil { return fmt.Errorf("failed to download install config from jumpbox: %w", err) @@ -125,7 +151,8 @@ func (b *GCPBootstrapper) recoverVault(dc *datacenter.DataCenter) error { } // UpdateInstallConfig writes the bootstrapped infrastructure into the primary data center's -// install config. +// install config. Secondary data centers go through updateInstallConfig directly, after their +// config and vault have been derived from the primary's. func (b *GCPBootstrapper) UpdateInstallConfig() error { b.ensureDataCenters() @@ -134,10 +161,7 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { func (b *GCPBootstrapper) updateInstallConfig(dc *datacenter.DataCenter) error { // Update install config with necessary values - dc.InstallConfig.Datacenter.ID = dc.ID - dc.InstallConfig.Datacenter.Name = dc.Name - dc.InstallConfig.Datacenter.City = "Karlsruhe" - dc.InstallConfig.Datacenter.CountryCode = "DE" + dc.InstallConfig.Datacenter = datacenterConfig(dc) // Each data center reads and writes its own vault. The installer resolves the vault from // secrets.baseDir, so sharing a directory would let one data center's ceph and kubernetes // steps overwrite another's credentials. @@ -157,16 +181,8 @@ func (b *GCPBootstrapper) updateInstallConfig(dc *datacenter.DataCenter) error { dc.InstallConfig.Registry.LoadContainerImages = true } - if dc.InstallConfig.Postgres.Primary == nil { - dc.InstallConfig.Postgres.Primary = &files.PostgresPrimaryConfig{ - Hostname: b.Env.PostgreSQLNode.GetName(), - } - } - - previousPrimaryIP := dc.InstallConfig.Postgres.Primary.IP - previousPrimaryHostname := dc.InstallConfig.Postgres.Primary.Hostname - dc.InstallConfig.Postgres.Primary.IP = b.Env.PostgreSQLNode.GetInternalIP() - dc.InstallConfig.Postgres.Primary.Hostname = b.Env.PostgreSQLNode.GetName() + previousPrimaryIP, previousPrimaryHostname := b.applyPostgresConfig(dc) + b.applyDataCenterTopology(dc) dc.InstallConfig.Ceph.CsiKubeletDir = "/var/lib/k0s/kubelet" // All data centers share the project's subnet; their Ceph clusters stay separate because @@ -291,7 +307,11 @@ func (b *GCPBootstrapper) updateInstallConfig(dc *datacenter.DataCenter) error { dc.InstallConfig.Codesphere.CustomDomains = files.CustomDomainsConfig{ CNameBaseDomain: dc.WorkspaceHostingBaseDomain, } - dc.InstallConfig.Codesphere.PublicIP = dc.ControlPlaneNodes[1].GetExternalIP() + if b.Env.MultiDC { + dc.InstallConfig.Codesphere.PublicIP = dc.PublicGatewayIP + } else { + dc.InstallConfig.Codesphere.PublicIP = dc.ControlPlaneNodes[1].GetExternalIP() + } dc.InstallConfig.Codesphere.DNSServers = []string{"8.8.8.8"} dc.InstallConfig.Codesphere.DeployConfig = bootstrap.DefaultCodesphereDeployConfig() dc.InstallConfig.Codesphere.Plans = bootstrap.DefaultCodespherePlans() @@ -409,7 +429,9 @@ func (b *GCPBootstrapper) updateInstallConfig(dc *datacenter.DataCenter) error { b.applyExternalLokiConfig(dc) b.applyPrometheusRemoteWriteConfig(dc) - if !dc.ExistingConfigUsed { + // A secondary data center always generates: its vault was seeded from the primary's, so the + // sentinels stop everything except its own ingress CA and cephadm key from being regenerated. + if !dc.ExistingConfigUsed || !dc.IsPrimary() { err := dc.ConfigManager.GenerateSecrets() if err != nil { return fmt.Errorf("failed to generate secrets: %w", err) @@ -420,6 +442,12 @@ func (b *GCPBootstrapper) updateInstallConfig(dc *datacenter.DataCenter) error { } } + if !dc.IsPrimary() { + if err := b.verifySecondaryDataCenterSecrets(b.primaryDC(), dc); err != nil { + return err + } + } + if b.Env.CentralOtelUsername != "" && b.Env.CentralOtelPassword != "" { if dc.InstallConfig.Cluster.Monitoring == nil { dc.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} @@ -466,6 +494,108 @@ func (b *GCPBootstrapper) updateInstallConfig(dc *datacenter.DataCenter) error { return nil } +// datacenterConfig describes a data center the way the install config does. All data centers of a +// bootstrapped instance live in the same GCP region, so city and country code are the same for all +// of them. +func datacenterConfig(dc *datacenter.DataCenter) files.DatacenterConfig { + return files.DatacenterConfig{ + ID: dc.ID, + Name: dc.Name, + City: "Karlsruhe", + CountryCode: "DE", + } +} + +// applyDataCenterTopology tells the platform about every data center of the installation. Without +// it, the installer defaults the list to the local data center, so each data center of a multi-DC +// instance would render as a single-data-center one. The list is identical in every data center's +// config; dataCenter stays the local one, so the platform still knows which data center it runs in. +// +// A single data center keeps the list unset and relies on the installer's default, so its config is +// unchanged from what OMS has always written. +func (b *GCPBootstrapper) applyDataCenterTopology(dc *datacenter.DataCenter) { + if len(b.Env.DataCenters) < 2 { + return + } + + all := make([]files.DatacenterConfig, 0, len(b.Env.DataCenters)) + for _, other := range b.Env.DataCenters { + all = append(all, datacenterConfig(other)) + } + dc.InstallConfig.DataCenters = all + // Clients land in the primary data center, which is also the one cs. resolves to. + dc.InstallConfig.DefaultDataCenterID = b.primaryDC().ID + + b.dropAvailableDataCentersOverride(dc) +} + +// dropAvailableDataCentersOverride removes the chart override OMS used to write before the +// installer supported dataCenters in the config. A recovered config may still carry it, where it +// would shadow the list above with a bare ID list. Any other override content is left alone. +func (b *GCPBootstrapper) dropAvailableDataCentersOverride(dc *datacenter.DataCenter) { + global, ok := dc.InstallConfig.Codesphere.Override["global"].(map[string]interface{}) + if !ok { + return + } + + delete(global, "availableDataCenters") + if len(global) == 0 { + delete(dc.InstallConfig.Codesphere.Override, "global") + } + if len(dc.InstallConfig.Codesphere.Override) == 0 { + dc.InstallConfig.Codesphere.Override = nil + } +} + +// applyPostgresConfig points the data center at its PostgreSQL server. The primary data center +// installs the server on its own node; every other data center connects to that same server as +// an external one and skips the postgres install step. +// +// Returns the primary IP and hostname the config held before, so regeneratePostgresCerts can +// tell whether the server's identity changed. +func (b *GCPBootstrapper) applyPostgresConfig(dc *datacenter.DataCenter) (previousIP, previousHostname string) { + if dc.ExternalPostgres { + dc.InstallConfig.Postgres = files.PostgresConfig{ + Mode: "external", + // The server certificate carries only an IP SAN, so the address must be the IP. + ServerAddress: b.Env.PostgreSQLNode.GetInternalIP(), + Port: sharedPostgresPort, + // The CA certificate lives in the config, not the vault, and cannot be re-derived + // from the CA key — so it has to be copied from the primary data center. + CACertPem: b.primaryDC().InstallConfig.Postgres.CACertPem, + Primary: nil, + Replica: nil, + } + b.skipInstallerStep(dc, "postgres") + + return "", "" + } + + if dc.InstallConfig.Postgres.Primary == nil { + dc.InstallConfig.Postgres.Primary = &files.PostgresPrimaryConfig{ + Hostname: b.Env.PostgreSQLNode.GetName(), + } + } + + previousIP = dc.InstallConfig.Postgres.Primary.IP + previousHostname = dc.InstallConfig.Postgres.Primary.Hostname + dc.InstallConfig.Postgres.Primary.IP = b.Env.PostgreSQLNode.GetInternalIP() + dc.InstallConfig.Postgres.Primary.Hostname = b.Env.PostgreSQLNode.GetName() + + return previousIP, previousHostname +} + +// skipInstallerStep persists a skipped installer step in the data center's config, so manual +// `oms install codesphere` re-runs on the jumpbox skip it too. +func (b *GCPBootstrapper) skipInstallerStep(dc *datacenter.DataCenter, step string) { + if dc.InstallConfig.Operations == nil { + dc.InstallConfig.Operations = &files.OperationsConfig{} + } + if !slices.Contains(dc.InstallConfig.Operations.Skip, step) { + dc.InstallConfig.Operations.Skip = append(dc.InstallConfig.Operations.Skip, step) + } +} + func (b *GCPBootstrapper) applySshProxyConfig(dc *datacenter.DataCenter) { dc.InstallConfig.PcApps = util.DeepMergeMaps(dc.InstallConfig.PcApps, files.ChartValues{ "applications": map[string]any{ @@ -533,8 +663,13 @@ func (b *GCPBootstrapper) applyPrometheusRemoteWriteConfig(dc *datacenter.DataCe } // regeneratePostgresCerts regenerates PostgreSQL TLS certificates when the IP/hostname -// changed or no private key was loaded from the vault. +// changed or no private key was loaded from the vault. It is a no-op for a data center that +// connects to an external server, since that server owns its own certificates. func (b *GCPBootstrapper) regeneratePostgresCerts(dc *datacenter.DataCenter, previousPrimaryIP, previousPrimaryHostname string) error { + if dc.InstallConfig.Postgres.Primary == nil { + return nil + } + vault := dc.ConfigManager.GetVault() primaryKeySecret := vault.GetSecret(files.SecretPostgresPrimaryServerKeyPem) primaryNeedsRegen := primaryKeySecret == nil || primaryKeySecret.File == nil || @@ -667,3 +802,92 @@ func (b *GCPBootstrapper) decryptVault(dc *datacenter.DataCenter, dst string) er return nil } + +// writeDataCenterConfig writes the data center's install config and vault, then places the +// encrypted vault and its age identity on the jumpbox. +func (b *GCPBootstrapper) writeDataCenterConfig(dc *datacenter.DataCenter) error { + err := b.stlog.Step(dc.StepName("Update install config"), func() error { + return b.updateInstallConfig(dc) + }) + if err != nil { + return fmt.Errorf("failed to update install config of data center %d: %w", dc.ID, err) + } + + err = b.stlog.Step(dc.StepName("Ensure age key"), func() error { + return b.ensureAgeKey(dc) + }) + if err != nil { + return fmt.Errorf("failed to ensure age key of data center %d: %w", dc.ID, err) + } + + err = b.stlog.Step(dc.StepName("Encrypt vault"), func() error { + return b.encryptVault(dc) + }) + if err != nil { + return fmt.Errorf("failed to encrypt vault of data center %d: %w", dc.ID, err) + } + + return nil +} + +// seedSecondaryDataCenter derives a secondary data center's config and vault from the primary +// one's, for the parts it does not already have. Secrets tied to the shared database and to +// cross-data-center authentication are copied verbatim; the per-cluster ones are dropped so +// GenerateSecrets regenerates them for this data center. +// +// Anything the data center already loaded from its own files is kept, so re-runs do not rotate a +// live installation's secrets. +func (b *GCPBootstrapper) seedSecondaryDataCenter(primary, dc *datacenter.DataCenter) error { + if dc.InstallConfig == nil { + config, err := secrets.DeriveDataCenterConfig(primary.InstallConfig) + if err != nil { + return fmt.Errorf("failed to derive config from data center %d: %w", primary.ID, err) + } + dc.ConfigManager.SetInstallConfig(config) + dc.InstallConfig = dc.ConfigManager.GetInstallConfig() + } + + if len(dc.ConfigManager.GetVault().Secrets) == 0 { + dc.ConfigManager.SetVault(secrets.DeriveDataCenterVault(primary.ConfigManager.GetVault())) + } + + return nil +} + +// verifySecondaryDataCenterSecrets fails the bootstrap if a secondary data center's secrets +// diverged from the primary's where they must match. Divergent PostgreSQL roles would let one +// data center's install rotate credentials the other one is using, and a divergent token key +// would break cross-data-center authentication — both only visible long after the fact. +func (b *GCPBootstrapper) verifySecondaryDataCenterSecrets(primary, dc *datacenter.DataCenter) error { + primaryVault := primary.ConfigManager.GetVault() + vault := dc.ConfigManager.GetVault() + + shared := []string{files.SecretTokenPrivateKey, files.SecretPostgresPassword} + for _, svc := range codesphere.PostgresServices { + shared = append(shared, files.PostgresUserSecretName(svc.Name), files.PostgresPasswordSecretName(svc.Name)) + } + for _, name := range shared { + expected := primaryVault.GetSecret(name) + if expected == nil { + continue + } + if !reflect.DeepEqual(vault.GetSecret(name), expected) { + return fmt.Errorf("secret %q of data center %d differs from the primary data center, but both use the same database", name, dc.ID) + } + } + + if dc.InstallConfig.Postgres.CACertPem == "" { + return fmt.Errorf("data center %d has no postgres CA certificate and could not verify the shared server", dc.ID) + } + + for _, name := range []string{files.SecretKubeConfig, files.SecretCephSshPrivateKey} { + if vault.GetSecret(name) == nil { + continue + } + if reflect.DeepEqual(vault.GetSecret(name), primaryVault.GetSecret(name)) { + return fmt.Errorf("secret %q of data center %d is the primary data center's, but they run separate clusters", name, dc.ID) + } + } + + return nil +} diff --git a/internal/bootstrap/gcp/install_config_test.go b/internal/bootstrap/gcp/install_config_test.go index 6ccd50d0..3e4b0c1d 100644 --- a/internal/bootstrap/gcp/install_config_test.go +++ b/internal/bootstrap/gcp/install_config_test.go @@ -344,6 +344,10 @@ var _ = Describe("Installconfig & Secrets", func() { Expect(bs.Env.InstallConfig.Datacenter.ID).To(Equal(1)) Expect(bs.Env.InstallConfig.Datacenter.Name).To(Equal("dev")) + // A single data center relies on the installer defaulting the topology to the + // local data center, so the config stays as it was before multi-DC support. + Expect(bs.Env.InstallConfig.DataCenters).To(BeEmpty()) + Expect(bs.Env.InstallConfig.DefaultDataCenterID).To(BeZero()) Expect(bs.Env.InstallConfig.Codesphere.Domain).To(Equal("cs.example.com")) Expect(bs.Env.InstallConfig.Codesphere.Features).To(Equal(map[string]bool{})) Expect(bs.Env.InstallConfig.Codesphere.Internal).To(Equal(gcp.DefaultInternalFlags))