From aacd865ca3f26262862ae3c774981bcaf0c7535b Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 14:59:46 +0200 Subject: [PATCH] refac(gcp): model the data centers of a bootstrap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until now a bootstrapped project was implicitly a single data center: its nodes, gateway IPs, config paths and domains all lived directly on CodesphereEnvironment. Multi-DC support needs more than one of each, so this introduces the DataCenter type that holds everything which must differ per data center, while project-level state (project, VPC, jumpbox, shared postgres node, registry) stays on the environment. BuildDataCenters derives the layout from the flags: one entry today, and with --multi-dc a second one that shares the first's PostgreSQL server. The primary data center keeps an empty resource-name suffix, so every name, path and domain a single-DC bootstrap produces is unchanged. Nothing consumes the layout yet — the callers are migrated in the following commits. Two mechanisms keep that migration safe: - ensureDataCenters derives the layout on first use and adopts state a caller passed through the legacy top-level environment fields, so every entry point works whether or not Bootstrap ran first, including infra files written before multi-DC support. - mirrorPrimaryDataCenter projects the primary data center back onto those fields before the infra file is written, so cleanup and restart-vms keep reading what they always have. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- internal/bootstrap/gcp/datacenter.go | 268 ++++++++++++++++++++++ internal/bootstrap/gcp/datacenter_test.go | 105 +++++++++ internal/bootstrap/gcp/gcp.go | 68 ++++-- internal/bootstrap/gcp/infrafile.go | 6 + 4 files changed, 424 insertions(+), 23 deletions(-) create mode 100644 internal/bootstrap/gcp/datacenter.go create mode 100644 internal/bootstrap/gcp/datacenter_test.go diff --git a/internal/bootstrap/gcp/datacenter.go b/internal/bootstrap/gcp/datacenter.go new file mode 100644 index 00000000..7ef4fbcc --- /dev/null +++ b/internal/bootstrap/gcp/datacenter.go @@ -0,0 +1,268 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package gcp + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" + "github.com/codesphere-cloud/oms/internal/installer/node" +) + +// primaryDatacenterID is the ID of the first data center. It stays 1 in both modes so +// single-DC bootstraps keep their existing dataCenter.id. +const primaryDatacenterID = 1 + +// DataCenter holds the state of one Codesphere data center inside a bootstrapped GCP project. +// Project-level state (the GCP project, VPC, jumpbox, shared postgres node and container +// registry) lives on CodesphereEnvironment; everything that must differ between data centers +// lives here. +type DataCenter struct { + ID int `json:"id"` + Name string `json:"name"` + // Suffix is appended to data-center-scoped GCP resource names. It is empty for the primary + // data center, so single-DC bootstraps keep the resource names they have always used. + Suffix string `json:"suffix"` + + ControlPlaneNodes []*node.Node `json:"control_plane_nodes"` + CephNodes []*node.Node `json:"ceph_nodes"` + + GatewayIP string `json:"gateway_ip"` + PublicGatewayIP string `json:"public_gateway_ip"` + SshProxyIP string `json:"ssh_proxy_ip"` + + // Local paths of the generated config and vault. + InstallConfigPath string `json:"-"` + SecretsFilePath string `json:"-"` + // Paths on the shared jumpbox. + RemoteConfigPath string `json:"remote_config_path"` + SecretsDir string `json:"secrets_dir"` + + WorkspaceHostingBaseDomain string `json:"workspace_hosting_base_domain"` + SshBaseDomain string `json:"ssh_base_domain"` + + // ExternalPostgres marks a data center that uses the primary data center's PostgreSQL + // server instead of installing its own. + ExternalPostgres bool `json:"external_postgres"` + + InstallConfig *files.RootConfig `json:"-"` + ExistingConfigUsed bool `json:"-"` + icg installer.InstallConfigManager `json:"-"` +} + +// IsPrimary reports whether this is the first data center of the installation. The primary data +// center owns the shared PostgreSQL server and the platform gateway that codesphere.domain +// resolves to. +func (dc *DataCenter) IsPrimary() bool { + return dc.Suffix == "" +} + +// ConfigManager returns the install config manager owning this data center's config and vault. +func (dc *DataCenter) ConfigManager() installer.InstallConfigManager { + return dc.icg +} + +// SetConfigManager assigns the install config manager for this data center. Exported for tests; +// Bootstrap assigns it via BuildDataCenters. +func (dc *DataCenter) SetConfigManager(icg installer.InstallConfigManager) { + dc.icg = icg +} + +// RemoteVaultPath returns the path of this data center's vault on the jumpbox. +func (dc *DataCenter) RemoteVaultPath() string { + return filepath.Join(dc.SecretsDir, "prod.vault.yaml") +} + +// RemoteAgeKeyPath returns the path of this data center's age identity on the jumpbox. +func (dc *DataCenter) RemoteAgeKeyPath() string { + return filepath.Join(dc.SecretsDir, "age_key.txt") +} + +// K0sConfigScriptPath returns the local filename of this data center's k0s configuration script. +func (dc *DataCenter) K0sConfigScriptPath() string { + return fmt.Sprintf("configure-k0s%s.sh", dc.Suffix) +} + +// StepName qualifies a bootstrap step name with the data center it applies to. Single-DC +// bootstraps keep their unqualified step names. +func (dc *DataCenter) StepName(name string) string { + if dc.Suffix == "" { + return name + } + return fmt.Sprintf("%s (dc %d)", name, dc.ID) +} + +// BuildDataCenters derives the data center layout from the bootstrap environment: a single +// entry in single-DC mode, and two entries in multi-DC mode where the second one shares the +// first one's PostgreSQL server. +func BuildDataCenters(env *CodesphereEnvironment, newICG func() installer.InstallConfigManager) []*DataCenter { + if !env.MultiDC { + // A single data center keeps honouring --datacenter-id. In multi-DC mode the IDs are + // derived instead, because they drive the per-data-center domains; validateMultiDC + // rejects the combination. + id := env.DatacenterID + if id == 0 { + id = primaryDatacenterID + } + return []*DataCenter{newDataCenter(env, id, "", newICG)} + } + + return []*DataCenter{ + newDataCenter(env, primaryDatacenterID, "", newICG), + newDataCenter(env, primaryDatacenterID+1, "-dc2", newICG), + } +} + +// ensureDataCenters makes sure the environment has a usable data center layout. It derives the +// layout on first use and gives every data center an install config manager, so any entry point +// works whether or not Bootstrap ran first. +func (b *GCPBootstrapper) ensureDataCenters() { + if len(b.Env.DataCenters) > 0 { + b.ensureConfigManagers() + return + } + + b.Env.DataCenters = BuildDataCenters(b.Env, nil) + b.adoptLegacyEnvFields() + b.ensureConfigManagers() +} + +// ensureConfigManagers gives every data center an install config manager. The primary one reuses +// the bootstrapper's, so a single-DC bootstrap behaves exactly as it did before multi-DC support. +// Data centers restored from an infra file arrive without a manager, since it is not serialised. +func (b *GCPBootstrapper) ensureConfigManagers() { + newICG := b.NewConfigManager + if newICG == nil { + newICG = installer.NewInstallConfigManager + } + + for i, dc := range b.Env.DataCenters { + if dc.ConfigManager() != nil { + continue + } + if i == 0 && b.icg != nil { + dc.SetConfigManager(b.icg) + continue + } + dc.SetConfigManager(newICG()) + } +} + +// adoptLegacyEnvFields moves state that a caller supplied through the legacy top-level +// environment fields into the primary data center. Environments loaded from an infra file written +// before multi-DC support carry the primary data center's nodes and IPs there. +func (b *GCPBootstrapper) adoptLegacyEnvFields() { + primary := b.Env.DataCenters[0] + if len(primary.ControlPlaneNodes) == 0 { + primary.ControlPlaneNodes = b.Env.ControlPlaneNodes + } + if len(primary.CephNodes) == 0 { + primary.CephNodes = b.Env.CephNodes + } + if primary.GatewayIP == "" { + primary.GatewayIP = b.Env.GatewayIP + } + if primary.PublicGatewayIP == "" { + primary.PublicGatewayIP = b.Env.PublicGatewayIP + } + if primary.SshProxyIP == "" { + primary.SshProxyIP = b.Env.SshProxyIP + } + if primary.InstallConfig == nil { + primary.InstallConfig = b.Env.InstallConfig + } + // A caller that supplied a config through the environment also tells us whether it is an + // existing one, which decides between generating and regenerating secrets. + if b.Env.ExistingConfigUsed { + primary.ExistingConfigUsed = true + } +} + +// mirrorPrimaryDataCenter projects the primary data center's state onto the legacy top-level +// environment fields. Those are what the infra file exposes to `cleanup` and `restart-vms`, and +// what infra files written before multi-DC support contain. The projection is one-way and never +// read back into a DataCenter. +func (b *GCPBootstrapper) mirrorPrimaryDataCenter() { + if len(b.Env.DataCenters) == 0 { + return + } + + primary := b.primaryDC() + b.Env.ControlPlaneNodes = primary.ControlPlaneNodes + b.Env.CephNodes = primary.CephNodes + b.Env.GatewayIP = primary.GatewayIP + b.Env.PublicGatewayIP = primary.PublicGatewayIP + b.Env.SshProxyIP = primary.SshProxyIP + b.Env.InstallConfig = primary.InstallConfig + b.Env.ExistingConfigUsed = primary.ExistingConfigUsed +} + +// newDataCenter builds one data center, deriving its resource names, file paths and domains +// from the environment and the data-center suffix. +func newDataCenter(env *CodesphereEnvironment, id int, suffix string, newICG func() installer.InstallConfigManager) *DataCenter { + name := env.DatacenterName + if name == "" { + name = "dev" + } + if suffix != "" { + // The k0s cluster is named codesphere-, so the names must differ. + name += suffix + } + + dc := &DataCenter{ + ID: id, + Name: name, + Suffix: suffix, + InstallConfigPath: dcSuffixedPath(env.InstallConfigPath, suffix), + SecretsFilePath: dcSuffixedPath(env.SecretsFilePath, suffix), + RemoteConfigPath: dcSuffixedPath(remoteInstallConfigPath, suffix), + SecretsDir: env.SecretsDir + suffix, + WorkspaceHostingBaseDomain: workspaceHostingBaseDomain(env, id), + SshBaseDomain: sshBaseDomain(env, id), + ExternalPostgres: suffix != "", + } + if newICG != nil { + dc.icg = newICG() + } + + return dc +} + +// workspaceHostingBaseDomain returns the domain workspaces of the given data center are served +// from. Single-DC installations keep ws.; multi-DC installations prefix it with the +// data center ID so each data center's public gateway gets its own name. +func workspaceHostingBaseDomain(env *CodesphereEnvironment, id int) string { + if !env.MultiDC { + return "ws." + env.BaseDomain + } + return fmt.Sprintf("%d.ws.%s", id, env.BaseDomain) +} + +// sshBaseDomain returns the domain the workspace SSH proxy of the given data center is served +// from, following the same scheme as workspaceHostingBaseDomain. +func sshBaseDomain(env *CodesphereEnvironment, id int) string { + if !env.MultiDC { + return "ssh.cs." + env.BaseDomain + } + return fmt.Sprintf("%d.ssh.cs.%s", id, env.BaseDomain) +} + +// dcSuffixedPath inserts the data-center suffix before the file extension, turning +// config.yaml into config-dc2.yaml and prod.vault.yaml into prod-dc2.vault.yaml. +func dcSuffixedPath(path, suffix string) string { + if suffix == "" { + return path + } + + dir, file := filepath.Split(path) + base, ext := file, "" + if idx := strings.Index(file, "."); idx > 0 { + base, ext = file[:idx], file[idx:] + } + + return filepath.Join(dir, base+suffix+ext) +} diff --git a/internal/bootstrap/gcp/datacenter_test.go b/internal/bootstrap/gcp/datacenter_test.go new file mode 100644 index 00000000..76522bbf --- /dev/null +++ b/internal/bootstrap/gcp/datacenter_test.go @@ -0,0 +1,105 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package gcp_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/codesphere-cloud/oms/internal/bootstrap/gcp" + "github.com/codesphere-cloud/oms/internal/installer" +) + +var _ = Describe("BuildDataCenters", func() { + newEnv := func(multiDC bool) *gcp.CodesphereEnvironment { + return &gcp.CodesphereEnvironment{ + MultiDC: multiDC, + BaseDomain: "example.com", + DatacenterName: "dev", + SecretsDir: "/etc/codesphere/secrets", + InstallConfigPath: "config.yaml", + SecretsFilePath: "prod.vault.yaml", + } + } + + Context("single data center", func() { + It("keeps the paths, secrets dir and domains a single-DC bootstrap has always used", func() { + dcs := gcp.BuildDataCenters(newEnv(false), installer.NewInstallConfigManager) + + Expect(dcs).To(HaveLen(1)) + dc := dcs[0] + Expect(dc.IsPrimary()).To(BeTrue()) + Expect(dc.ID).To(Equal(1)) + Expect(dc.Name).To(Equal("dev")) + Expect(dc.Suffix).To(BeEmpty()) + Expect(dc.InstallConfigPath).To(Equal("config.yaml")) + Expect(dc.SecretsFilePath).To(Equal("prod.vault.yaml")) + Expect(dc.RemoteConfigPath).To(Equal("/etc/codesphere/config.yaml")) + Expect(dc.SecretsDir).To(Equal("/etc/codesphere/secrets")) + Expect(dc.RemoteVaultPath()).To(Equal("/etc/codesphere/secrets/prod.vault.yaml")) + Expect(dc.RemoteAgeKeyPath()).To(Equal("/etc/codesphere/secrets/age_key.txt")) + Expect(dc.K0sConfigScriptPath()).To(Equal("configure-k0s.sh")) + Expect(dc.WorkspaceHostingBaseDomain).To(Equal("ws.example.com")) + Expect(dc.SshBaseDomain).To(Equal("ssh.cs.example.com")) + Expect(dc.ExternalPostgres).To(BeFalse()) + Expect(dc.StepName("Encrypt vault")).To(Equal("Encrypt vault")) + }) + }) + + Context("multi data center", func() { + var dcs []*gcp.DataCenter + + BeforeEach(func() { + dcs = gcp.BuildDataCenters(newEnv(true), installer.NewInstallConfigManager) + }) + + It("builds two data centers with the second sharing the first's postgres", func() { + Expect(dcs).To(HaveLen(2)) + Expect(dcs[0].ExternalPostgres).To(BeFalse()) + Expect(dcs[1].ExternalPostgres).To(BeTrue()) + }) + + It("leaves the primary data center's resource names unsuffixed", func() { + Expect(dcs[0].Suffix).To(BeEmpty()) + Expect(dcs[0].InstallConfigPath).To(Equal("config.yaml")) + Expect(dcs[0].SecretsDir).To(Equal("/etc/codesphere/secrets")) + }) + + It("gives the secondary data center its own name, paths and secrets dir", func() { + dc := dcs[1] + Expect(dc.IsPrimary()).To(BeFalse()) + Expect(dc.ID).To(Equal(2)) + // The k0s cluster is named codesphere-, so the names must differ. + Expect(dc.Name).To(Equal("dev-dc2")) + Expect(dc.InstallConfigPath).To(Equal("config-dc2.yaml")) + Expect(dc.SecretsFilePath).To(Equal("prod-dc2.vault.yaml")) + Expect(dc.RemoteConfigPath).To(Equal("/etc/codesphere/config-dc2.yaml")) + // A separate secrets dir, so the installer cannot overwrite the primary's kubeconfig + // and ceph credentials through config.secrets.baseDir. + Expect(dc.SecretsDir).To(Equal("/etc/codesphere/secrets-dc2")) + Expect(dc.RemoteVaultPath()).To(Equal("/etc/codesphere/secrets-dc2/prod.vault.yaml")) + Expect(dc.RemoteAgeKeyPath()).To(Equal("/etc/codesphere/secrets-dc2/age_key.txt")) + Expect(dc.K0sConfigScriptPath()).To(Equal("configure-k0s-dc2.sh")) + Expect(dc.StepName("Encrypt vault")).To(Equal("Encrypt vault (dc 2)")) + }) + + It("scopes the workspace and ssh domains per data center", func() { + Expect(dcs[0].WorkspaceHostingBaseDomain).To(Equal("1.ws.example.com")) + Expect(dcs[0].SshBaseDomain).To(Equal("1.ssh.cs.example.com")) + Expect(dcs[1].WorkspaceHostingBaseDomain).To(Equal("2.ws.example.com")) + Expect(dcs[1].SshBaseDomain).To(Equal("2.ssh.cs.example.com")) + }) + + It("gives each data center its own config manager", func() { + Expect(dcs[0].ConfigManager()).NotTo(BeIdenticalTo(dcs[1].ConfigManager())) + }) + }) + + It("falls back to the dev datacenter name", func() { + env := newEnv(false) + env.DatacenterName = "" + + Expect(gcp.BuildDataCenters(env, installer.NewInstallConfigManager)[0].Name).To(Equal("dev")) + }) +}) diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index 19eb5c30..e8ef40ce 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -96,18 +96,36 @@ type GCPBootstrapper struct { NodeClient node.NodeClient PortalClient portal.Portal GitHubClient github.GitHubClient + // NewConfigManager creates the install config manager of a data center. Each data center + // owns its own config and vault, so multi-DC bootstraps need more than one. + NewConfigManager func() installer.InstallConfigManager +} + +// primaryDC returns the first data center, which owns the shared PostgreSQL server and the +// platform gateway that codesphere.domain resolves to. +func (b *GCPBootstrapper) primaryDC() *DataCenter { + return b.Env.DataCenters[0] } type CodesphereEnvironment struct { - ProjectID string `json:"project_id"` - ProjectTTL string `json:"project_ttl"` - ProjectName string `json:"project_name"` - DNSProjectID string `json:"dns_project_id"` - Jumpbox *node.Node `json:"jumpbox"` - PostgreSQLNode *node.Node `json:"postgres_node"` - ControlPlaneNodes []*node.Node `json:"control_plane_nodes"` - CephNodes []*node.Node `json:"ceph_nodes"` - ContainerRegistryURL string `json:"-"` + ProjectID string `json:"project_id"` + ProjectTTL string `json:"project_ttl"` + ProjectName string `json:"project_name"` + DNSProjectID string `json:"dns_project_id"` + Jumpbox *node.Node `json:"jumpbox"` + PostgreSQLNode *node.Node `json:"postgres_node"` + // MultiDC bootstraps two data centers that share the PostgreSQL server but run separate + // Kubernetes and Ceph clusters. + MultiDC bool `json:"multi_dc"` + // DataCenters holds the per-data-center state. It always has at least one entry. + DataCenters []*DataCenter `json:"datacenters"` + // Mirrors of DataCenters[0], written for infra files consumed by cleanup and restart-vms. + ControlPlaneNodes []*node.Node `json:"control_plane_nodes"` + CephNodes []*node.Node `json:"ceph_nodes"` + // ContainerRegistryURL is the resolved registry server all data centers pull images from. + ContainerRegistryURL string `json:"container_registry_url,omitempty"` + RegistryUsername string `json:"-"` + RegistryPassword string `json:"-"` ExistingConfigUsed bool `json:"-"` InstallVersion string `json:"install_version"` InstallLocal string `json:"install_local"` @@ -181,10 +199,13 @@ type CodesphereEnvironment struct { SSHPrivateKeyPath string `json:"-"` DatacenterID int `json:"-"` DatacenterName string `json:"-"` - CustomPgIP string `json:"custom_pg_ip"` - Region string `json:"region"` - Zone string `json:"zone"` - DNSZoneName string `json:"dns_zone_name"` + // DatacenterIDExplicit records whether --datacenter-id was set on the command line. The + // value alone cannot distinguish the default 1 from an explicit 1. + DatacenterIDExplicit bool `json:"-"` + CustomPgIP string `json:"custom_pg_ip"` + Region string `json:"region"` + Zone string `json:"zone"` + DNSZoneName string `json:"dns_zone_name"` // Test user creation CreateTestUser bool `json:"-"` @@ -208,16 +229,17 @@ func NewGCPBootstrapper( gitHubClient github.GitHubClient, ) (*GCPBootstrapper, error) { return &GCPBootstrapper{ - ctx: ctx, - stlog: stlog, - fw: fw, - icg: icg, - GCPClient: gcpClient, - Env: CodesphereEnv, - NodeClient: sshRunner, - PortalClient: portalClient, - Time: time, - GitHubClient: gitHubClient, + ctx: ctx, + stlog: stlog, + fw: fw, + icg: icg, + GCPClient: gcpClient, + Env: CodesphereEnv, + NodeClient: sshRunner, + PortalClient: portalClient, + Time: time, + GitHubClient: gitHubClient, + NewConfigManager: installer.NewInstallConfigManager, }, nil } diff --git a/internal/bootstrap/gcp/infrafile.go b/internal/bootstrap/gcp/infrafile.go index 72378f9f..f7f6d855 100644 --- a/internal/bootstrap/gcp/infrafile.go +++ b/internal/bootstrap/gcp/infrafile.go @@ -40,6 +40,12 @@ func LoadInfraFile(fw util.FileIO, infraFilePath string) (CodesphereEnvironment, // WriteInfraFile writes details about the bootstrapped codesphere environment into a file. func (b *GCPBootstrapper) WriteInfraFile() error { + b.ensureDataCenters() + + // The legacy top-level node and IP fields are what cleanup and restart-vms read, so keep + // them in sync with the primary data center before serialising. + b.mirrorPrimaryDataCenter() + envBytes, err := json.MarshalIndent(b.Env, "", " ") if err != nil { return fmt.Errorf("failed to marshal codesphere env: %w", err)