From f8e00c79f4585d4333e695123ed02adb821971c6 Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 15:08:56 +0200 Subject: [PATCH] refac(gcp): thread the data center through install config generation Config and vault generation worked on b.Env.InstallConfig and the single b.icg. Both become per data center: every step takes the DataCenter it generates for and reads its nodes, IPs, paths and config manager from there. The exported Ensure*/Update* entry points keep their signatures and operate on the primary data center, so step names and existing callers are unchanged. Two details worth pointing out: - secrets.baseDir is now the data center's own directory. The installer resolves the vault it reads and writes back from that path, so sharing it between data centers would let one data center's kubernetes and ceph steps overwrite the other's kubeconfig and Ceph credentials. - the local config, vault and remote paths come from the data center, so a second data center writes config-dc2.yaml rather than overwriting config.yaml. Pure refactoring: a single-data-center bootstrap produces the same config and vault as before. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- internal/bootstrap/gcp/install_config.go | 360 +++++++++++++---------- 1 file changed, 204 insertions(+), 156 deletions(-) diff --git a/internal/bootstrap/gcp/install_config.go b/internal/bootstrap/gcp/install_config.go index 83ebdf0a..94adb979 100644 --- a/internal/bootstrap/gcp/install_config.go +++ b/internal/bootstrap/gcp/install_config.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/codesphere-cloud/oms/internal/bootstrap" + "github.com/codesphere-cloud/oms/internal/bootstrap/datacenter" "github.com/codesphere-cloud/oms/internal/installer/files" "github.com/codesphere-cloud/oms/internal/installer/secrets" "github.com/codesphere-cloud/oms/internal/util" @@ -16,47 +17,55 @@ const ( remoteInstallConfigPath string = "/etc/codesphere/config.yaml" ) -// EnsureInstallConfig uses the local config or recovers it from an existing jumpbox if desired. -// Else it applies the minimal profile to a new config. +// EnsureInstallConfig prepares the primary data center's install config. func (b *GCPBootstrapper) EnsureInstallConfig() error { + b.ensureDataCenters() + + return b.ensureInstallConfig(b.primaryDC()) +} + +// ensureInstallConfig uses the data center's local config or recovers it from an existing +// jumpbox if desired. Else it applies the minimal profile to a new config. +func (b *GCPBootstrapper) ensureInstallConfig(dc *datacenter.DataCenter) error { // recovery will overwrite local config or create a new file if b.Env.RecoverConfig { - err := b.recoverConfig() + err := b.recoverConfig(dc) if err != nil { return fmt.Errorf("failed to recover config: %w", err) } } - if b.fw.Exists(b.Env.InstallConfigPath) { - if err := b.loadVaultForConfigTemplating(); err != nil { + if b.fw.Exists(dc.InstallConfigPath) { + if err := b.loadVaultForConfigTemplating(dc); err != nil { return fmt.Errorf("failed to load vault templating: %w", err) } - err := b.icg.LoadInstallConfigFromFile(b.Env.InstallConfigPath) + err := dc.ConfigManager.LoadInstallConfigFromFile(dc.InstallConfigPath) if err != nil { return fmt.Errorf("failed to load config file: %w", err) } - b.Env.ExistingConfigUsed = true + dc.ExistingConfigUsed = true } else { - err := b.icg.ApplyProfile("minimal") + err := dc.ConfigManager.ApplyProfile("minimal") if err != nil { return fmt.Errorf("failed to apply profile: %w", err) } } - b.Env.InstallConfig = b.icg.GetInstallConfig() + dc.InstallConfig = dc.ConfigManager.GetInstallConfig() + b.mirrorPrimaryDataCenter() return nil } -func (b *GCPBootstrapper) loadVaultForConfigTemplating() error { - if !b.fw.Exists(b.Env.SecretsFilePath) { +func (b *GCPBootstrapper) loadVaultForConfigTemplating(dc *datacenter.DataCenter) error { + if !b.fw.Exists(dc.SecretsFilePath) { return nil } // during bootstrapping, the vault is not yet encrpyted - if err := b.icg.LoadVaultFromUnecryptedFile(b.Env.SecretsFilePath); err != nil { + if err := dc.ConfigManager.LoadVaultFromUnecryptedFile(dc.SecretsFilePath); err != nil { return fmt.Errorf("failed to load vault from file: %w", err) } @@ -66,7 +75,7 @@ func (b *GCPBootstrapper) loadVaultForConfigTemplating() error { // recoverConfig downloads the config and secrets from the jumpbox if it exists. // Since recovery is done when the project or VMs are not ensured, we need to search for the jumpbox IP first. // Returns an error if project or jumpbox does not exist or downloading fails. -func (b *GCPBootstrapper) recoverConfig() error { +func (b *GCPBootstrapper) recoverConfig(dc *datacenter.DataCenter) error { existingProject, err := b.GCPClient.GetProjectByName(b.Env.FolderID, b.Env.ProjectName) if err != nil { return fmt.Errorf("failed to find gcp project for config recovery: %w", err) @@ -79,12 +88,12 @@ func (b *GCPBootstrapper) recoverConfig() error { } b.Env.Jumpbox = jumpbox - err = b.Env.Jumpbox.NodeClient.DownloadFile(jumpbox, remoteInstallConfigPath, b.Env.InstallConfigPath) + err = b.Env.Jumpbox.NodeClient.DownloadFile(jumpbox, dc.RemoteConfigPath, dc.InstallConfigPath) if err != nil { return fmt.Errorf("failed to download install config from jumpbox: %w", err) } - err = b.recoverVault() + err = b.recoverVault(dc) if err != nil { return fmt.Errorf("failed to recover vault: %w", err) } @@ -93,8 +102,8 @@ func (b *GCPBootstrapper) recoverConfig() error { } // recoverVault unencrypts the secrets file on the jumpbox and download the file to the local destination -func (b *GCPBootstrapper) recoverVault() error { - const vaultCopyPath string = "/tmp/prod.vault.yaml" +func (b *GCPBootstrapper) recoverVault(dc *datacenter.DataCenter) error { + vaultCopyPath := fmt.Sprintf("/tmp/prod%s.vault.yaml", dc.Suffix) defer func() { err := b.Env.Jumpbox.RunSSHCommand("root", "rm -f "+vaultCopyPath) if err != nil { @@ -102,12 +111,12 @@ func (b *GCPBootstrapper) recoverVault() error { } }() - err := b.decryptVault(vaultCopyPath) + err := b.decryptVault(dc, vaultCopyPath) if err != nil { return fmt.Errorf("failed to create decrypted vault for recovery: %w", err) } - err = b.Env.Jumpbox.NodeClient.DownloadFile(b.Env.Jumpbox, vaultCopyPath, b.Env.SecretsFilePath) + err = b.Env.Jumpbox.NodeClient.DownloadFile(b.Env.Jumpbox, vaultCopyPath, dc.SecretsFilePath) if err != nil { return fmt.Errorf("failed to download secrets file from jumpbox: %w", err) } @@ -115,50 +124,60 @@ func (b *GCPBootstrapper) recoverVault() error { return nil } +// UpdateInstallConfig writes the bootstrapped infrastructure into the primary data center's +// install config. func (b *GCPBootstrapper) UpdateInstallConfig() error { + b.ensureDataCenters() + + return b.updateInstallConfig(b.primaryDC()) +} + +func (b *GCPBootstrapper) updateInstallConfig(dc *datacenter.DataCenter) error { // Update install config with necessary values - b.Env.InstallConfig.Datacenter.ID = b.Env.DatacenterID - if b.Env.DatacenterName == "" { - b.Env.DatacenterName = "dev" - } - b.Env.InstallConfig.Datacenter.Name = b.Env.DatacenterName - b.Env.InstallConfig.Datacenter.City = "Karlsruhe" - b.Env.InstallConfig.Datacenter.CountryCode = "DE" - b.Env.InstallConfig.Secrets.BaseDir = b.Env.SecretsDir + dc.InstallConfig.Datacenter.ID = dc.ID + dc.InstallConfig.Datacenter.Name = dc.Name + dc.InstallConfig.Datacenter.City = "Karlsruhe" + dc.InstallConfig.Datacenter.CountryCode = "DE" + // Each data center reads and writes its own vault. The installer resolves the vault from + // secrets.baseDir, so sharing a directory would let one data center's ceph and kubernetes + // steps overwrite another's credentials. + dc.InstallConfig.Secrets.BaseDir = dc.SecretsDir if b.Env.RegistryType != RegistryTypeGitHub { - b.Env.InstallConfig.Registry.ReplaceImagesInBom = true - b.Env.InstallConfig.Registry.LoadContainerImages = true + dc.InstallConfig.Registry.ReplaceImagesInBom = true + dc.InstallConfig.Registry.LoadContainerImages = true } - if b.Env.InstallConfig.Postgres.Primary == nil { - b.Env.InstallConfig.Postgres.Primary = &files.PostgresPrimaryConfig{ + if dc.InstallConfig.Postgres.Primary == nil { + dc.InstallConfig.Postgres.Primary = &files.PostgresPrimaryConfig{ Hostname: b.Env.PostgreSQLNode.GetName(), } } - previousPrimaryIP := b.Env.InstallConfig.Postgres.Primary.IP - previousPrimaryHostname := b.Env.InstallConfig.Postgres.Primary.Hostname - b.Env.InstallConfig.Postgres.Primary.IP = b.Env.PostgreSQLNode.GetInternalIP() - b.Env.InstallConfig.Postgres.Primary.Hostname = b.Env.PostgreSQLNode.GetName() + previousPrimaryIP := dc.InstallConfig.Postgres.Primary.IP + previousPrimaryHostname := dc.InstallConfig.Postgres.Primary.Hostname + dc.InstallConfig.Postgres.Primary.IP = b.Env.PostgreSQLNode.GetInternalIP() + dc.InstallConfig.Postgres.Primary.Hostname = b.Env.PostgreSQLNode.GetName() - b.Env.InstallConfig.Ceph.CsiKubeletDir = "/var/lib/k0s/kubelet" - b.Env.InstallConfig.Ceph.NodesSubnet = "10.10.0.0/20" - b.Env.InstallConfig.Ceph.Hosts = []files.CephHost{ + dc.InstallConfig.Ceph.CsiKubeletDir = "/var/lib/k0s/kubelet" + // All data centers share the project's subnet; their Ceph clusters stay separate because + // each has its own hosts, monitors and FSID. + dc.InstallConfig.Ceph.NodesSubnet = vpcSubnetCIDR + dc.InstallConfig.Ceph.Hosts = []files.CephHost{ { - Hostname: b.Env.CephNodes[0].GetName(), + Hostname: dc.CephNodes[0].GetName(), IsMaster: true, - IPAddress: b.Env.CephNodes[0].GetInternalIP(), + IPAddress: dc.CephNodes[0].GetInternalIP(), }, { - Hostname: b.Env.CephNodes[1].GetName(), - IPAddress: b.Env.CephNodes[1].GetInternalIP(), + Hostname: dc.CephNodes[1].GetName(), + IPAddress: dc.CephNodes[1].GetInternalIP(), }, { - Hostname: b.Env.CephNodes[2].GetName(), - IPAddress: b.Env.CephNodes[2].GetInternalIP(), + Hostname: dc.CephNodes[2].GetName(), + IPAddress: dc.CephNodes[2].GetInternalIP(), }, } - b.Env.InstallConfig.Ceph.OSDs = []files.CephOSD{ + dc.InstallConfig.Ceph.OSDs = []files.CephOSD{ { SpecID: "default", Placement: files.CephPlacement{ @@ -175,48 +194,48 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { }, } - b.Env.InstallConfig.Kubernetes = files.KubernetesConfig{ + dc.InstallConfig.Kubernetes = files.KubernetesConfig{ ManagedByCodesphere: true, - APIServerHost: b.Env.ControlPlaneNodes[0].GetInternalIP(), + APIServerHost: dc.ControlPlaneNodes[0].GetInternalIP(), ControlPlanes: []files.K8sNode{ { - IPAddress: b.Env.ControlPlaneNodes[0].GetInternalIP(), + IPAddress: dc.ControlPlaneNodes[0].GetInternalIP(), }, }, Workers: []files.K8sNode{ { - IPAddress: b.Env.ControlPlaneNodes[0].GetInternalIP(), + IPAddress: dc.ControlPlaneNodes[0].GetInternalIP(), }, { - IPAddress: b.Env.ControlPlaneNodes[1].GetInternalIP(), + IPAddress: dc.ControlPlaneNodes[1].GetInternalIP(), }, { - IPAddress: b.Env.ControlPlaneNodes[2].GetInternalIP(), + IPAddress: dc.ControlPlaneNodes[2].GetInternalIP(), }, }, } - b.Env.InstallConfig.Cluster.Kyverno = &files.KyvernoConfig{ + dc.InstallConfig.Cluster.Kyverno = &files.KyvernoConfig{ Enabled: false, } - b.Env.InstallConfig.Cluster.Gateway.ServiceType = "LoadBalancer" - b.Env.InstallConfig.Cluster.Gateway.Annotations = map[string]string{ - "cloud.google.com/load-balancer-ipv4": b.Env.GatewayIP, + dc.InstallConfig.Cluster.Gateway.ServiceType = "LoadBalancer" + dc.InstallConfig.Cluster.Gateway.Annotations = map[string]string{ + "cloud.google.com/load-balancer-ipv4": dc.GatewayIP, } - b.Env.InstallConfig.Cluster.PublicGateway.ServiceType = "LoadBalancer" - b.Env.InstallConfig.Cluster.PublicGateway.Annotations = map[string]string{ - "cloud.google.com/load-balancer-ipv4": b.Env.PublicGatewayIP, + dc.InstallConfig.Cluster.PublicGateway.ServiceType = "LoadBalancer" + dc.InstallConfig.Cluster.PublicGateway.Annotations = map[string]string{ + "cloud.google.com/load-balancer-ipv4": dc.PublicGatewayIP, } - b.applySshProxyConfig() + b.applySshProxyConfig(dc) dnsProject := b.Env.DNSProjectID if b.Env.DNSProjectID == "" { dnsProject = b.Env.ProjectID } - b.Env.InstallConfig.Cluster.Certificates.Override = map[string]interface{}{ + dc.InstallConfig.Cluster.Certificates.Override = map[string]interface{}{ "issuers": map[string]interface{}{ "letsEncryptHttp": map[string]interface{}{ "enabled": !b.Env.GoogleACMEIssuer, @@ -248,26 +267,28 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { } acmeConfig.Server = "https://dv.acme-v02.api.pki.goog/directory" acmeConfig.EABKeyID = keyID - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAcmeEabMacKey, Fields: &files.SecretFields{Password: b64MacKey}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAcmeEabMacKey, Fields: &files.SecretFields{Password: b64MacKey}}) } - b.Env.InstallConfig.Codesphere.CertIssuer = files.CertIssuerConfig{ + dc.InstallConfig.Codesphere.CertIssuer = files.CertIssuerConfig{ Type: "acme", Acme: acmeConfig, } - b.Env.InstallConfig.Codesphere.Domain = "cs." + b.Env.BaseDomain - b.Env.InstallConfig.Codesphere.WorkspaceHostingBaseDomain = "ws." + b.Env.BaseDomain - b.Env.InstallConfig.Codesphere.PublicIP = b.Env.ControlPlaneNodes[1].GetExternalIP() - b.Env.InstallConfig.Codesphere.CustomDomains = files.CustomDomainsConfig{ - CNameBaseDomain: "ws." + b.Env.BaseDomain, + // The platform is served from one domain shared by all data centers, while workspaces and + // custom domains resolve to the data center hosting them. + dc.InstallConfig.Codesphere.Domain = "cs." + b.Env.BaseDomain + dc.InstallConfig.Codesphere.WorkspaceHostingBaseDomain = dc.WorkspaceHostingBaseDomain + dc.InstallConfig.Codesphere.CustomDomains = files.CustomDomainsConfig{ + CNameBaseDomain: dc.WorkspaceHostingBaseDomain, } - b.Env.InstallConfig.Codesphere.DNSServers = []string{"8.8.8.8"} - b.Env.InstallConfig.Codesphere.DeployConfig = bootstrap.DefaultCodesphereDeployConfig() - b.Env.InstallConfig.Codesphere.Plans = bootstrap.DefaultCodespherePlans() + dc.InstallConfig.Codesphere.PublicIP = dc.ControlPlaneNodes[1].GetExternalIP() + dc.InstallConfig.Codesphere.DNSServers = []string{"8.8.8.8"} + dc.InstallConfig.Codesphere.DeployConfig = bootstrap.DefaultCodesphereDeployConfig() + dc.InstallConfig.Codesphere.Plans = bootstrap.DefaultCodespherePlans() - b.Env.InstallConfig.Codesphere.GitProviders = &files.GitProvidersConfig{} + dc.InstallConfig.Codesphere.GitProviders = &files.GitProvidersConfig{} if b.Env.GitHubAppName != "" && b.Env.GitHubAppClientID != "" && b.Env.GitHubAppClientSecret != "" { - b.Env.InstallConfig.Codesphere.GitProviders.GitHub = &files.GitProviderConfig{ + dc.InstallConfig.Codesphere.GitProviders.GitHub = &files.GitProviderConfig{ Enabled: true, URL: "https://github.com", API: files.APIConfig{ @@ -282,11 +303,11 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { InstallationURI: "https://github.com/apps/" + b.Env.GitHubAppName + "/installations/new", }, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGithubAppsClientId, Fields: &files.SecretFields{Password: b.Env.GitHubAppClientID}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGithubAppsClientSecret, Fields: &files.SecretFields{Password: b.Env.GitHubAppClientSecret}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGithubAppsClientId, Fields: &files.SecretFields{Password: b.Env.GitHubAppClientID}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGithubAppsClientSecret, Fields: &files.SecretFields{Password: b.Env.GitHubAppClientSecret}}) } if b.Env.GitLabAppClientID != "" && b.Env.GitLabAppClientSecret != "" { - b.Env.InstallConfig.Codesphere.GitProviders.GitLab = &files.GitProviderConfig{ + dc.InstallConfig.Codesphere.GitProviders.GitLab = &files.GitProviderConfig{ Enabled: true, URL: "https://gitlab.com", API: files.APIConfig{ @@ -300,11 +321,11 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { RedirectURI: "https://cs." + b.Env.BaseDomain + "/ide/auth/gitlab/callback", }, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGitlabAppClientId, Fields: &files.SecretFields{Password: b.Env.GitLabAppClientID}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGitlabAppClientSecret, Fields: &files.SecretFields{Password: b.Env.GitLabAppClientSecret}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGitlabAppClientId, Fields: &files.SecretFields{Password: b.Env.GitLabAppClientID}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretGitlabAppClientSecret, Fields: &files.SecretFields{Password: b.Env.GitLabAppClientSecret}}) } if b.Env.BitbucketAppClientID != "" && b.Env.BitbucketAppClientSecret != "" { - b.Env.InstallConfig.Codesphere.GitProviders.Bitbucket = &files.GitProviderConfig{ + dc.InstallConfig.Codesphere.GitProviders.Bitbucket = &files.GitProviderConfig{ Enabled: true, URL: "https://bitbucket.org", API: files.APIConfig{ @@ -318,11 +339,11 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { RedirectURI: "https://cs." + b.Env.BaseDomain + "/ide/auth/bitbucket/callback", }, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretBitbucketAppsClientId, Fields: &files.SecretFields{Password: b.Env.BitbucketAppClientID}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretBitbucketAppsClientSecret, Fields: &files.SecretFields{Password: b.Env.BitbucketAppClientSecret}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretBitbucketAppsClientId, Fields: &files.SecretFields{Password: b.Env.BitbucketAppClientID}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretBitbucketAppsClientSecret, Fields: &files.SecretFields{Password: b.Env.BitbucketAppClientSecret}}) } if b.Env.AzureDevOpsAppClientID != "" && b.Env.AzureDevOpsAppClientSecret != "" { - b.Env.InstallConfig.Codesphere.GitProviders.AzureDevOps = &files.GitProviderConfig{ + dc.InstallConfig.Codesphere.GitProviders.AzureDevOps = &files.GitProviderConfig{ Enabled: true, URL: "https://dev.azure.com", API: files.APIConfig{ @@ -337,15 +358,15 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { Scope: "openid offline_access https://app.vssps.visualstudio.com/vso.code_full", }, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAzureDevOpsAppClientId, Fields: &files.SecretFields{Password: b.Env.AzureDevOpsAppClientID}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAzureDevOpsAppClientSecret, Fields: &files.SecretFields{Password: b.Env.AzureDevOpsAppClientSecret}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAzureDevOpsAppClientId, Fields: &files.SecretFields{Password: b.Env.AzureDevOpsAppClientID}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretAzureDevOpsAppClientSecret, Fields: &files.SecretFields{Password: b.Env.AzureDevOpsAppClientSecret}}) } if b.Env.OidcIssuerURL != "" && b.Env.OidcClientID != "" && b.Env.OidcClientSecret != "" { name := b.Env.OidcProviderName if name == "" { name = "OIDC" } - b.Env.InstallConfig.Codesphere.OAuth = &files.OAuthProvidersConfig{ + dc.InstallConfig.Codesphere.OAuth = &files.OAuthProvidersConfig{ Oidc: &files.OidcOAuthProvider{ Type: "oidc", Enabled: true, @@ -354,12 +375,12 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { Scopes: []string{"openid", "profile", "email"}, }, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOidcClientId, Fields: &files.SecretFields{Password: b.Env.OidcClientID}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOidcClientSecret, Fields: &files.SecretFields{Password: b.Env.OidcClientSecret}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOidcClientId, Fields: &files.SecretFields{Password: b.Env.OidcClientID}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOidcClientSecret, Fields: &files.SecretFields{Password: b.Env.OidcClientSecret}}) } if b.Env.CentralOtelPassword != "" || b.Env.LocalTraceEndpoint != "" { - b.Env.InstallConfig.Codesphere.TelemetryExport = &files.TelemetryExport{ + dc.InstallConfig.Codesphere.TelemetryExport = &files.TelemetryExport{ RemoteEndpoint: b.Env.CentralOtelEndpoint, RemoteExport: b.Env.CentralOtelPassword != "", Traces: b.Env.LocalTraceEndpoint != "", @@ -368,71 +389,75 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { } } - b.Env.InstallConfig.Codesphere.Internal = b.Env.InternalFlags - b.Env.InstallConfig.Codesphere.Preview = util.StringSliceToBoolMap(b.Env.PreviewFlags) - b.Env.InstallConfig.Codesphere.Features = util.StringSliceToBoolMap(b.Env.FeatureFlags) + dc.InstallConfig.Codesphere.Internal = b.Env.InternalFlags + dc.InstallConfig.Codesphere.Preview = util.StringSliceToBoolMap(b.Env.PreviewFlags) + dc.InstallConfig.Codesphere.Features = util.StringSliceToBoolMap(b.Env.FeatureFlags) // Only set when the flag is provided so a recovered config keeps its value on re-runs. if b.Env.ClusterAdminEmail != "" { - b.Env.InstallConfig.Codesphere.ClusterAdminEmail = b.Env.ClusterAdminEmail + dc.InstallConfig.Codesphere.ClusterAdminEmail = b.Env.ClusterAdminEmail } - b.applyExternalLokiConfig() - b.applyPrometheusRemoteWriteConfig() + b.applyExternalLokiConfig(dc) + b.applyPrometheusRemoteWriteConfig(dc) - if !b.Env.ExistingConfigUsed { - err := b.icg.GenerateSecrets() + if !dc.ExistingConfigUsed { + err := dc.ConfigManager.GenerateSecrets() if err != nil { return fmt.Errorf("failed to generate secrets: %w", err) } } else { - if err := b.regeneratePostgresCerts(previousPrimaryIP, previousPrimaryHostname); err != nil { + if err := b.regeneratePostgresCerts(dc, previousPrimaryIP, previousPrimaryHostname); err != nil { return err } } if b.Env.CentralOtelUsername != "" && b.Env.CentralOtelPassword != "" { - if b.Env.InstallConfig.Cluster.Monitoring == nil { - b.Env.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} + if dc.InstallConfig.Cluster.Monitoring == nil { + dc.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} } - b.Env.InstallConfig.Cluster.Monitoring.CentralOtelExport = &files.CentralOtelConfig{ + dc.InstallConfig.Cluster.Monitoring.CentralOtelExport = &files.CentralOtelConfig{ Enabled: true, Username: b.Env.CentralOtelUsername, Password: b.Env.CentralOtelPassword, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretCentralOtelCreds, Fields: &files.SecretFields{Username: b.Env.CentralOtelUsername, Password: b.Env.CentralOtelPassword}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretCentralOtelCreds, Fields: &files.SecretFields{Username: b.Env.CentralOtelUsername, Password: b.Env.CentralOtelPassword}}) } if b.Env.OpenBaoURI != "" { - b.Env.InstallConfig.Codesphere.OpenBao = &files.OpenBaoConfig{ + dc.InstallConfig.Codesphere.OpenBao = &files.OpenBaoConfig{ Engine: b.Env.OpenBaoEngine, URI: b.Env.OpenBaoURI, User: b.Env.OpenBaoUser, } - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOpenBaoPassword, Fields: &files.SecretFields{Password: b.Env.OpenBaoPassword}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretOpenBaoPassword, Fields: &files.SecretFields{Password: b.Env.OpenBaoPassword}}) } - if err := b.icg.WriteInstallConfig(b.Env.InstallConfigPath, true); err != nil { + if err := dc.ConfigManager.WriteInstallConfig(dc.InstallConfigPath, true); err != nil { return fmt.Errorf("failed to write config file: %w", err) } - if err := b.icg.WriteVault(b.Env.SecretsFilePath, true); err != nil { + if err := dc.ConfigManager.WriteVault(dc.SecretsFilePath, true); err != nil { return fmt.Errorf("failed to write vault file: %w", err) } - err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.InstallConfigPath, remoteInstallConfigPath) + // CopyFile creates the destination directory, so a secondary data center's secrets + // directory does not need to exist yet. + err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, dc.InstallConfigPath, dc.RemoteConfigPath) if err != nil { return fmt.Errorf("failed to copy install config to jumpbox: %w", err) } - err = b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.SecretsFilePath, b.Env.SecretsDir+"/prod.vault.yaml") + err = b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, dc.SecretsFilePath, dc.RemoteVaultPath()) if err != nil { return fmt.Errorf("failed to copy secrets file to jumpbox: %w", err) } + b.mirrorPrimaryDataCenter() + return nil } -func (b *GCPBootstrapper) applySshProxyConfig() { - b.Env.InstallConfig.PcApps = util.DeepMergeMaps(b.Env.InstallConfig.PcApps, files.ChartValues{ +func (b *GCPBootstrapper) applySshProxyConfig(dc *datacenter.DataCenter) { + dc.InstallConfig.PcApps = util.DeepMergeMaps(dc.InstallConfig.PcApps, files.ChartValues{ "applications": map[string]any{ "ssh-workspace-proxy": map[string]any{ "enabled": true, @@ -440,9 +465,9 @@ func (b *GCPBootstrapper) applySshProxyConfig() { "service": map[string]any{ "enabled": true, "type": "LoadBalancer", - "loadBalancerIP": b.Env.SshProxyIP, + "loadBalancerIP": dc.SshProxyIP, "annotations": map[string]any{ - "cloud.google.com/load-balancer-ipv4": b.Env.SshProxyIP, + "cloud.google.com/load-balancer-ipv4": dc.SshProxyIP, }, }, }, @@ -451,16 +476,16 @@ func (b *GCPBootstrapper) applySshProxyConfig() { }) } -func (b *GCPBootstrapper) applyExternalLokiConfig() { +func (b *GCPBootstrapper) applyExternalLokiConfig(dc *datacenter.DataCenter) { if b.Env.ExternalLokiEndpoint == "" { return } - if b.Env.InstallConfig.Cluster.Monitoring == nil { - b.Env.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} + if dc.InstallConfig.Cluster.Monitoring == nil { + dc.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} } - if b.Env.InstallConfig.Cluster.Monitoring.GrafanaAlloy == nil { - b.Env.InstallConfig.Cluster.Monitoring.GrafanaAlloy = &files.GrafanaAlloyConfig{} + if dc.InstallConfig.Cluster.Monitoring.GrafanaAlloy == nil { + dc.InstallConfig.Cluster.Monitoring.GrafanaAlloy = &files.GrafanaAlloyConfig{} } loki := &files.LokiConnectionConfig{ @@ -469,42 +494,42 @@ func (b *GCPBootstrapper) applyExternalLokiConfig() { Password: b.Env.ExternalLokiSecret, } - b.Env.InstallConfig.Cluster.Monitoring.GrafanaAlloy.Enabled = true - b.Env.InstallConfig.Cluster.Monitoring.GrafanaAlloy.Loki = loki - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretLokiGatewayBasicAuthPassword, Fields: &files.SecretFields{Password: b.Env.ExternalLokiSecret}}) + dc.InstallConfig.Cluster.Monitoring.GrafanaAlloy.Enabled = true + dc.InstallConfig.Cluster.Monitoring.GrafanaAlloy.Loki = loki + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretLokiGatewayBasicAuthPassword, Fields: &files.SecretFields{Password: b.Env.ExternalLokiSecret}}) } -func (b *GCPBootstrapper) applyPrometheusRemoteWriteConfig() { +func (b *GCPBootstrapper) applyPrometheusRemoteWriteConfig(dc *datacenter.DataCenter) { if b.Env.PrometheusRemoteWriteURL == "" { return } - if b.Env.InstallConfig.Cluster.Monitoring == nil { - b.Env.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} + if dc.InstallConfig.Cluster.Monitoring == nil { + dc.InstallConfig.Cluster.Monitoring = &files.MonitoringConfig{} } - if b.Env.InstallConfig.Cluster.Monitoring.Prometheus == nil { - b.Env.InstallConfig.Cluster.Monitoring.Prometheus = &files.PrometheusConfig{} + if dc.InstallConfig.Cluster.Monitoring.Prometheus == nil { + dc.InstallConfig.Cluster.Monitoring.Prometheus = &files.PrometheusConfig{} } - if b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite == nil { - b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite = &files.RemoteWriteConfig{} + if dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite == nil { + dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite = &files.RemoteWriteConfig{} } - b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Enabled = true - b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Url = b.Env.PrometheusRemoteWriteURL - b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.ClusterName = b.Env.DatacenterName - b.Env.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Username = b.Env.PrometheusRemoteWriteUser - b.icg.GetVault().SetSecret(files.SecretEntry{Name: "promRemoteWritePassword", Fields: &files.SecretFields{Password: b.Env.PrometheusRemoteWritePassword}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: "promRemoteWriteUser", Fields: &files.SecretFields{Password: b.Env.PrometheusRemoteWriteUser}}) + dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Enabled = true + dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Url = b.Env.PrometheusRemoteWriteURL + dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.ClusterName = dc.Name + dc.InstallConfig.Cluster.Monitoring.Prometheus.RemoteWrite.Username = b.Env.PrometheusRemoteWriteUser + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: "promRemoteWritePassword", Fields: &files.SecretFields{Password: b.Env.PrometheusRemoteWritePassword}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: "promRemoteWriteUser", Fields: &files.SecretFields{Password: b.Env.PrometheusRemoteWriteUser}}) } // regeneratePostgresCerts regenerates PostgreSQL TLS certificates when the IP/hostname // changed or no private key was loaded from the vault. -func (b *GCPBootstrapper) regeneratePostgresCerts(previousPrimaryIP, previousPrimaryHostname string) error { - vault := b.icg.GetVault() +func (b *GCPBootstrapper) regeneratePostgresCerts(dc *datacenter.DataCenter, previousPrimaryIP, previousPrimaryHostname string) error { + vault := dc.ConfigManager.GetVault() primaryKeySecret := vault.GetSecret(files.SecretPostgresPrimaryServerKeyPem) primaryNeedsRegen := primaryKeySecret == nil || primaryKeySecret.File == nil || - previousPrimaryIP != b.Env.InstallConfig.Postgres.Primary.IP || - previousPrimaryHostname != b.Env.InstallConfig.Postgres.Primary.Hostname + previousPrimaryIP != dc.InstallConfig.Postgres.Primary.IP || + previousPrimaryHostname != dc.InstallConfig.Postgres.Primary.Hostname if primaryNeedsRegen { caSecret := vault.GetSecret(files.SecretPostgresCaKeyPem) @@ -513,9 +538,9 @@ func (b *GCPBootstrapper) regeneratePostgresCerts(previousPrimaryIP, previousPri } primaryKeyPEM, primaryCertPEM, err := secrets.GenerateServerCertificate( caSecret.File.Content, - b.Env.InstallConfig.Postgres.CACertPem, - b.Env.InstallConfig.Postgres.Primary.Hostname, - []string{b.Env.InstallConfig.Postgres.Primary.IP}) + dc.InstallConfig.Postgres.CACertPem, + dc.InstallConfig.Postgres.Primary.Hostname, + []string{dc.InstallConfig.Postgres.Primary.IP}) if err != nil { return fmt.Errorf("failed to generate primary server certificate: %w", err) } @@ -523,9 +548,9 @@ func (b *GCPBootstrapper) regeneratePostgresCerts(previousPrimaryIP, previousPri return fmt.Errorf("primary PostgreSQL cert/key validation failed: %w", err) } vault.SetSecret(files.SecretEntry{Name: files.SecretPostgresPrimaryServerKeyPem, File: &files.SecretFile{Name: "primary.key", Content: primaryKeyPEM}}) - b.Env.InstallConfig.Postgres.Primary.SSLConfig.ServerCertPem = primaryCertPEM + dc.InstallConfig.Postgres.Primary.SSLConfig.ServerCertPem = primaryCertPEM } - if b.Env.InstallConfig.Postgres.Replica != nil { + if dc.InstallConfig.Postgres.Replica != nil { replicaKeySecret := vault.GetSecret(files.SecretPostgresReplicaServerKeyPem) if replicaKeySecret == nil || replicaKeySecret.File == nil { caSecret := vault.GetSecret(files.SecretPostgresCaKeyPem) @@ -534,9 +559,9 @@ func (b *GCPBootstrapper) regeneratePostgresCerts(previousPrimaryIP, previousPri } replicaKeyPEM, replicaCertPEM, err := secrets.GenerateServerCertificate( caSecret.File.Content, - b.Env.InstallConfig.Postgres.CACertPem, - b.Env.InstallConfig.Postgres.Replica.Name, - []string{b.Env.InstallConfig.Postgres.Replica.IP}) + dc.InstallConfig.Postgres.CACertPem, + dc.InstallConfig.Postgres.Replica.Name, + []string{dc.InstallConfig.Postgres.Replica.IP}) if err != nil { return fmt.Errorf("failed to generate replica server certificate: %w", err) } @@ -544,19 +569,25 @@ func (b *GCPBootstrapper) regeneratePostgresCerts(previousPrimaryIP, previousPri return fmt.Errorf("replica PostgreSQL cert/key validation failed: %w", err) } vault.SetSecret(files.SecretEntry{Name: files.SecretPostgresReplicaServerKeyPem, File: &files.SecretFile{Name: "replica.key", Content: replicaKeyPEM}}) - b.Env.InstallConfig.Postgres.Replica.SSLConfig.ServerCertPem = replicaCertPEM + dc.InstallConfig.Postgres.Replica.SSLConfig.ServerCertPem = replicaCertPEM } } return nil } +// EnsureAgeKey generates the primary data center's age identity on the jumpbox. func (b *GCPBootstrapper) EnsureAgeKey() error { - hasKey := b.Env.Jumpbox.NodeClient.HasFile(b.Env.Jumpbox, b.Env.SecretsDir+"/age_key.txt") - if hasKey { + b.ensureDataCenters() + + return b.ensureAgeKey(b.primaryDC()) +} + +func (b *GCPBootstrapper) ensureAgeKey(dc *datacenter.DataCenter) error { + if b.Env.Jumpbox.NodeClient.HasFile(b.Env.Jumpbox, dc.RemoteAgeKeyPath()) { return nil } - err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("mkdir -p %s; age-keygen -o %s/age_key.txt", b.Env.SecretsDir, b.Env.SecretsDir)) + err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("mkdir -p %s; age-keygen -o %s", dc.SecretsDir, dc.RemoteAgeKeyPath())) if err != nil { return fmt.Errorf("failed to generate age key on jumpbox: %w", err) } @@ -564,24 +595,41 @@ func (b *GCPBootstrapper) EnsureAgeKey() error { return nil } +// EnsureSecrets loads the primary data center's vault if it already exists locally. func (b *GCPBootstrapper) EnsureSecrets() error { - if b.fw.Exists(b.Env.SecretsFilePath) { - err := b.icg.LoadVaultFromUnecryptedFile(b.Env.SecretsFilePath) + b.ensureDataCenters() + + return b.ensureSecrets(b.primaryDC()) +} + +func (b *GCPBootstrapper) ensureSecrets(dc *datacenter.DataCenter) error { + if b.fw.Exists(dc.SecretsFilePath) { + err := dc.ConfigManager.LoadVaultFromUnecryptedFile(dc.SecretsFilePath) if err != nil { return fmt.Errorf("failed to load vault file: %w", err) } } - b.Env.Secrets = b.icg.GetVault() + if dc.IsPrimary() { + b.Env.Secrets = dc.ConfigManager.GetVault() + } + b.mirrorPrimaryDataCenter() return nil } +// EncryptVault encrypts the primary data center's vault on the jumpbox. func (b *GCPBootstrapper) EncryptVault() error { - err := b.Env.Jumpbox.RunSSHCommand("root", "cp "+b.Env.SecretsDir+"/prod.vault.yaml{,.bak}") + b.ensureDataCenters() + + return b.encryptVault(b.primaryDC()) +} + +func (b *GCPBootstrapper) encryptVault(dc *datacenter.DataCenter) error { + err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("cp %s{,.bak}", dc.RemoteVaultPath())) if err != nil { return fmt.Errorf("failed backup vault on jumpbox: %w", err) } - err = b.Env.Jumpbox.RunSSHCommand("root", "sops --encrypt --in-place --age $(age-keygen -y "+b.Env.SecretsDir+"/age_key.txt) "+b.Env.SecretsDir+"/prod.vault.yaml") + err = b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("sops --encrypt --in-place --age $(age-keygen -y %s) %s", dc.RemoteAgeKeyPath(), dc.RemoteVaultPath())) if err != nil { return fmt.Errorf("failed to encrypt vault on jumpbox: %w", err) } @@ -589,10 +637,10 @@ func (b *GCPBootstrapper) EncryptVault() error { return nil } -// decryptVault creates an unencrypted copy of the vault in dst on the jumpbox +// decryptVault creates an unencrypted copy of the data center's vault in dst on the jumpbox. // Make sure to delete the unencrypted file when not needed anymore. -func (b *GCPBootstrapper) decryptVault(dst string) error { - err := b.Env.Jumpbox.RunSSHCommand("root", "cp "+b.Env.SecretsDir+"/prod.vault.yaml "+dst) +func (b *GCPBootstrapper) decryptVault(dc *datacenter.DataCenter, dst string) error { + err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("cp %s %s", dc.RemoteVaultPath(), dst)) if err != nil { return fmt.Errorf("failed to create tmp vault on jumpbox: %w", err) } @@ -602,7 +650,7 @@ func (b *GCPBootstrapper) decryptVault(dst string) error { return fmt.Errorf("failed to make vault file readable only for root on jumpbox: %w", err) } - err = b.Env.Jumpbox.RunSSHCommand("root", "SOPS_AGE_KEY_FILE="+b.Env.SecretsDir+"/age_key.txt sops --decrypt --in-place "+dst) + err = b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("SOPS_AGE_KEY_FILE=%s sops --decrypt --in-place %s", dc.RemoteAgeKeyPath(), dst)) if err != nil { return fmt.Errorf("failed to decrypt vault on jumpbox: %w", err) }