From 58f659e78da94ac50aca45d5a360bccb72648120 Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 15:03:33 +0200 Subject: [PATCH] feat(gcp): configure hosts, gateways and installs per data center MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrates the infrastructure steps from the single implicit data center to the layout: root login and host configuration run over every data center's nodes, each data center reserves its own gateway, public gateway and SSH proxy IP under a suffixed name, and each gets its own k0s configuration script patching its own gateway services. InstallCodesphere and RunK0sConfigScript now loop over the data centers in ascending order and log a step per data center. The order matters once there is more than one: the primary data center's install creates the database, roles and schema the others reuse, and a k0s script can only patch gateway services an install has already created. The install command moves into the exported InstallCommand, since the CLI prints it for the operator when the bootstrap does not install itself, and it now names the data center's own config, vault and age key. EnsureHostsConfigured also creates /etc/codesphere/secrets up front on every node: the installer uploads a data center's age key to that fixed path but only creates its own configured secrets.baseDir, so for a data center whose baseDir differs the upload target would not exist. Behaviour for a single data center is unchanged — same VM names, same IP names, same script, same install command. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- internal/bootstrap/gcp/gcp.go | 211 +++++++++++++++++++++++------ internal/bootstrap/gcp/gcp_test.go | 65 +++++++++ 2 files changed, 233 insertions(+), 43 deletions(-) diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index e8ef40ce..72aaa895 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "log" - "path/filepath" "slices" "strings" "time" @@ -36,6 +35,24 @@ const ( RegistryTypeGitHub RegistryType = "github" ) +// remoteK0sConfigScriptPath is where each data center's k0s configuration script is placed on +// that data center's first control plane node. Data centers have separate nodes, so the path can +// be the same for all of them. +const remoteK0sConfigScriptPath = "/root/configure-k0s.sh" + +// installerNodeSecretsDir is where the Codesphere installer uploads a data center's age key on +// every one of that data center's nodes. The path is fixed even though the installer reads the key +// from the data center's own secrets.baseDir on the jumpbox, and the installer only creates +// baseDir on the node — so for a data center whose baseDir differs, the upload target would not +// exist. Creating it up front is harmless: data centers have separate nodes, so a node only ever +// holds its own data center's key. +const installerNodeSecretsDir = "/etc/codesphere/secrets" + +// vpcSubnetCIDR is the range of the project's single subnet, shared by all data centers. It is +// also each data center's ceph.nodesSubnet; their Ceph clusters stay separate because each has +// its own hosts, monitors and FSID. +const vpcSubnetCIDR = "10.10.0.0/20" + // CheckOMSManagedLabel checks if the given labels map indicates an OMS-managed project. // A project is considered OMS-managed if it has the 'oms-managed' label set to "true". func CheckOMSManagedLabel(labels map[string]string) bool { @@ -107,6 +124,30 @@ func (b *GCPBootstrapper) primaryDC() *DataCenter { return b.Env.DataCenters[0] } +// allNodes returns every node of the project: the jumpbox, the shared postgres node and all +// data centers' Ceph and k0s nodes. +func (b *GCPBootstrapper) allNodes() []*node.Node { + nodes := []*node.Node{b.Env.Jumpbox, b.Env.PostgreSQLNode} + for _, dc := range b.Env.DataCenters { + nodes = append(nodes, dc.ControlPlaneNodes...) + nodes = append(nodes, dc.CephNodes...) + } + + return nodes +} + +// clusterNodes returns every Ceph and k0s node of all data centers, i.e. all nodes except the +// jumpbox and the shared postgres node. +func (b *GCPBootstrapper) clusterNodes() []*node.Node { + nodes := []*node.Node{} + for _, dc := range b.Env.DataCenters { + nodes = append(nodes, dc.ControlPlaneNodes...) + nodes = append(nodes, dc.CephNodes...) + } + + return nodes +} + type CodesphereEnvironment struct { ProjectID string `json:"project_id"` ProjectTTL string `json:"project_ttl"` @@ -373,12 +414,14 @@ func (b *GCPBootstrapper) Bootstrap() error { } if b.Env.InstallVersion != "" || b.Env.InstallLocal != "" { - err = b.stlog.Step("Install Codesphere", b.InstallCodesphere) + err = b.InstallCodesphere() if err != nil { return fmt.Errorf("failed to install Codesphere: %w", err) } - err = b.stlog.Step("Run k0s config script", b.RunK0sConfigScript) + // Every data center is installed before any k0s script runs, so a script never patches + // gateway services an install has yet to create. + err = b.RunK0sConfigScript() if err != nil { return fmt.Errorf("failed to run k0s config script: %w", err) } @@ -393,8 +436,11 @@ func (b *GCPBootstrapper) Bootstrap() error { return nil } -// createTestUser creates a test user in the PostgreSQL instance using the testuser package and logs the credentials. +// createTestUser creates a test user in the shared PostgreSQL instance using the testuser package +// and logs the credentials. The user's team is homed in the primary data center. func (b *GCPBootstrapper) createTestUser() error { + b.ensureDataCenters() + if b.Env.PostgreSQLNode == nil { return fmt.Errorf("postgres node not found in bootstrap environment") } @@ -404,10 +450,11 @@ func (b *GCPBootstrapper) createTestUser() error { return fmt.Errorf("postgres node has no external IP") } - if b.Env.InstallConfig == nil { + primary := b.primaryDC() + if primary.InstallConfig == nil { return fmt.Errorf("install config not found in bootstrap environment") } - pgPasswordSecret := b.icg.GetVault().GetSecret(files.SecretPostgresPassword) + pgPasswordSecret := primary.ConfigManager().GetVault().GetSecret(files.SecretPostgresPassword) if pgPasswordSecret == nil || pgPasswordSecret.Fields == nil { return fmt.Errorf("postgres admin password not found in vault") } @@ -420,7 +467,7 @@ func (b *GCPBootstrapper) createTestUser() error { Password: pgPassword, DBName: testuser.DefaultDBName, SSLMode: "require", - DatacenterID: b.Env.DatacenterID, + DatacenterID: primary.ID, }) if err != nil { return err @@ -707,7 +754,7 @@ func (b *GCPBootstrapper) EnsureFirewallRules() error { Allowed: []*computepb.Allowed{ {IPProtocol: protoString("all")}, }, - SourceRanges: []string{"10.10.0.0/20"}, + SourceRanges: []string{vpcSubnetCIDR}, Description: protoString("Allow all internal traffic"), } err = b.GCPClient.CreateFirewallRule(b.Env.ProjectID, internalRule) @@ -770,19 +817,34 @@ func (b *GCPBootstrapper) EnsureFirewallRules() error { return nil } -// EnsureGatewayIPAddresses reserves the static external IP addresses for the ingress -// controllers of the cluster (gateway and public gateway) and the SSH workspace proxy. +// EnsureGatewayIPAddresses reserves the static external IP addresses of every data center: the +// ingress controllers of its cluster (gateway and public gateway) and its SSH workspace proxy. func (b *GCPBootstrapper) EnsureGatewayIPAddresses() error { + b.ensureDataCenters() + + for _, dc := range b.Env.DataCenters { + if err := b.ensureGatewayIPAddresses(dc); err != nil { + return err + } + } + b.mirrorPrimaryDataCenter() + + return nil +} + +// ensureGatewayIPAddresses reserves one data center's static external IP addresses. Their names +// carry the data-center suffix, so the primary data center keeps the unsuffixed names. +func (b *GCPBootstrapper) ensureGatewayIPAddresses(dc *DataCenter) error { var err error - b.Env.GatewayIP, err = b.EnsureExternalIP("gateway") + dc.GatewayIP, err = b.EnsureExternalIP("gateway" + dc.Suffix) if err != nil { return fmt.Errorf("failed to ensure gateway IP: %w", err) } - b.Env.PublicGatewayIP, err = b.EnsureExternalIP("public-gateway") + dc.PublicGatewayIP, err = b.EnsureExternalIP("public-gateway" + dc.Suffix) if err != nil { return fmt.Errorf("failed to ensure public gateway IP: %w", err) } - b.Env.SshProxyIP, err = b.EnsureExternalIP("ssh-proxy") + dc.SshProxyIP, err = b.EnsureExternalIP("ssh-proxy" + dc.Suffix) if err != nil { return fmt.Errorf("failed to ensure ssh proxy IP: %w", err) } @@ -823,14 +885,9 @@ func (b *GCPBootstrapper) EnsureExternalIP(name string) (string, error) { } func (b *GCPBootstrapper) EnsureRootLoginEnabled() error { - allNodes := []*node.Node{ - b.Env.Jumpbox, - } - allNodes = append(allNodes, b.Env.ControlPlaneNodes...) - allNodes = append(allNodes, b.Env.PostgreSQLNode) - allNodes = append(allNodes, b.Env.CephNodes...) + b.ensureDataCenters() - for _, node := range allNodes { + for _, node := range b.allNodes() { err := b.stlog.Substep(fmt.Sprintf("Ensuring root login enabled on %s", node.GetName()), func() error { return b.ensureRootLoginEnabledInNode(node) }) @@ -916,8 +973,9 @@ func (b *GCPBootstrapper) EnsureOmsInstalled() (err error) { } func (b *GCPBootstrapper) EnsureHostsConfigured() error { - allNodes := append(b.Env.ControlPlaneNodes, b.Env.PostgreSQLNode) - allNodes = append(allNodes, b.Env.CephNodes...) + b.ensureDataCenters() + + allNodes := append([]*node.Node{b.Env.PostgreSQLNode}, b.clusterNodes()...) for _, node := range allNodes { if !node.HasInotifyWatchesConfigured() { @@ -932,6 +990,25 @@ func (b *GCPBootstrapper) EnsureHostsConfigured() error { return fmt.Errorf("failed to configure memory map on %s: %w", node.GetName(), err) } } + err := node.RunSSHCommand("root", "mkdir -p "+installerNodeSecretsDir) + if err != nil { + return fmt.Errorf("failed to create secrets directory on %s: %w", node.GetName(), err) + } + } + + // A secondary data center's secrets directory differs from the fixed path above, so create that + // one too on its own nodes. Nodes belong to exactly one data center, so no node gets a foreign + // data center's directory. + for _, dc := range b.Env.DataCenters { + if dc.SecretsDir == installerNodeSecretsDir { + continue + } + for _, n := range append(append([]*node.Node{}, dc.ControlPlaneNodes...), dc.CephNodes...) { + err := n.RunSSHCommand("root", "mkdir -p "+dc.SecretsDir) + if err != nil { + return fmt.Errorf("failed to create secrets directory on %s: %w", n.GetName(), err) + } + } } return nil @@ -1079,15 +1156,42 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error { return nil } +// InstallCodesphere installs Codesphere into every data center from the shared jumpbox, in +// ascending data center order. The order matters: the primary data center's install creates the +// database, roles and schema that the secondary ones reuse. func (b *GCPBootstrapper) InstallCodesphere() error { + b.ensureDataCenters() + fullPackageFilename, err := b.ensureCodespherePackageOnJumpbox() if err != nil { return fmt.Errorf("failed to ensure Codesphere package on jumpbox: %w", err) } - err = b.runInstallCommand(fullPackageFilename) - if err != nil { - return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err) + for _, dc := range b.Env.DataCenters { + err = b.stlog.Step(dc.StepName("Install Codesphere"), func() error { + return b.runInstallCommand(dc, fullPackageFilename) + }) + if err != nil { + return fmt.Errorf("failed to install Codesphere from jumpbox (data center %d): %w", dc.ID, err) + } + } + + return nil +} + +// RunK0sConfigScript runs every data center's k0s configuration script on its first control +// plane node. It requires that data center's Codesphere install to have completed, since the +// script patches the gateway services the install creates. +func (b *GCPBootstrapper) RunK0sConfigScript() error { + b.ensureDataCenters() + + for _, dc := range b.Env.DataCenters { + err := b.stlog.Step(dc.StepName("Run k0s config script"), func() error { + return b.runK0sConfigScript(dc) + }) + if err != nil { + return err + } } return nil @@ -1128,11 +1232,16 @@ func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() (string, error) { return fullPackageFilename, nil } -func (b *GCPBootstrapper) runInstallCommand(packageFilename string) error { - b.stlog.Logf("Installing Codesphere...") - installCmd := fmt.Sprintf("oms install codesphere -c /etc/codesphere/config.yaml -k %s/age_key.txt --vault %s -p %s%s", - b.Env.SecretsDir, filepath.Join(b.Env.SecretsDir, "prod.vault.yaml"), packageFilename, b.generateSkipStepsArg()) - return b.Env.Jumpbox.RunSSHCommand("root", installCmd) +func (b *GCPBootstrapper) runInstallCommand(dc *DataCenter, packageFilename string) error { + b.stlog.Logf("Installing Codesphere in data center %d...", dc.ID) + return b.Env.Jumpbox.RunSSHCommand("root", b.InstallCommand(dc, packageFilename)) +} + +// InstallCommand returns the command that installs Codesphere into the given data center from +// the jumpbox. It is also printed for the operator when the bootstrap does not install itself. +func (b *GCPBootstrapper) InstallCommand(dc *DataCenter, packageFilename string) string { + return fmt.Sprintf("oms install codesphere -c %s -k %s --vault %s -p %s%s", + dc.RemoteConfigPath, dc.RemoteAgeKeyPath(), dc.RemoteVaultPath(), packageFilename, b.generateSkipStepsArg()) } func (b *GCPBootstrapper) generateSkipStepsArg() string { @@ -1147,7 +1256,21 @@ func (b *GCPBootstrapper) generateSkipStepsArg() string { return " -s " + strings.Join(skipSteps, ",") } +// GenerateK0sConfigScript writes and uploads the k0s cloud-provider configuration script of +// every data center to that data center's first control plane node. func (b *GCPBootstrapper) GenerateK0sConfigScript() error { + b.ensureDataCenters() + + for _, dc := range b.Env.DataCenters { + if err := b.generateK0sConfigScript(dc); err != nil { + return err + } + } + + return nil +} + +func (b *GCPBootstrapper) generateK0sConfigScript(dc *DataCenter) error { script := `#!/bin/bash cat < cloud.conf @@ -1215,14 +1338,14 @@ $KUBECTL apply -f https://raw.githubusercontent.com/kubernetes/cloud-provider-gc $KUBECTL apply -f cc-deployment.yaml # set loadBalancerIP for public-gateway-controller and gateway-controller -$KUBECTL patch svc public-gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + b.Env.PublicGatewayIP + `'"}}' -$KUBECTL patch svc gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + b.Env.GatewayIP + `'"}}' +$KUBECTL patch svc public-gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + dc.PublicGatewayIP + `'"}}' +$KUBECTL patch svc gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + dc.GatewayIP + `'"}}' sed -i 's/k0scontroller/k0scontroller --enable-cloud-provider/g' /etc/systemd/system/k0scontroller.service -ssh -o StrictHostKeyChecking=no root@` + b.Env.ControlPlaneNodes[1].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" +ssh -o StrictHostKeyChecking=no root@` + dc.ControlPlaneNodes[1].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" -ssh -o StrictHostKeyChecking=no root@` + b.Env.ControlPlaneNodes[2].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" +ssh -o StrictHostKeyChecking=no root@` + dc.ControlPlaneNodes[2].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" systemctl daemon-reload systemctl restart k0scontroller @@ -1231,25 +1354,27 @@ systemctl restart k0scontroller // --enable-cloud-provider on worker nodes systemd file /etc/systemd/system/k0sworker.service // in addition on the first node: /etc/systemd/system/k0scontroller.service the flag --enable-cloud-provider - err := b.fw.WriteFile("configure-k0s.sh", []byte(script), 0755) + localScript := dc.K0sConfigScriptPath() + err := b.fw.WriteFile(localScript, []byte(script), 0755) if err != nil { - return fmt.Errorf("failed to write configure-k0s.sh: %w", err) + return fmt.Errorf("failed to write %s: %w", localScript, err) } - err = b.Env.ControlPlaneNodes[0].NodeClient.CopyFile(b.Env.ControlPlaneNodes[0], "configure-k0s.sh", "/root/configure-k0s.sh") + controller := dc.ControlPlaneNodes[0] + err = controller.NodeClient.CopyFile(controller, localScript, remoteK0sConfigScriptPath) if err != nil { - return fmt.Errorf("failed to copy configure-k0s.sh to control plane node: %w", err) + return fmt.Errorf("failed to copy %s to control plane node: %w", localScript, err) } - err = b.Env.ControlPlaneNodes[0].RunSSHCommand("root", "chmod +x /root/configure-k0s.sh") + err = controller.RunSSHCommand("root", "chmod +x "+remoteK0sConfigScriptPath) if err != nil { - return fmt.Errorf("failed to make configure-k0s.sh executable on control plane node: %w", err) + return fmt.Errorf("failed to make %s executable: %w", localScript, err) } return nil } -func (b *GCPBootstrapper) RunK0sConfigScript() error { - err := b.Env.ControlPlaneNodes[0].RunSSHCommand("root", "/root/configure-k0s.sh") +func (b *GCPBootstrapper) runK0sConfigScript(dc *DataCenter) error { + err := dc.ControlPlaneNodes[0].RunSSHCommand("root", remoteK0sConfigScriptPath) if err != nil { - return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err) + return fmt.Errorf("failed to configure k0s in data center %d: %w", dc.ID, err) } return nil diff --git a/internal/bootstrap/gcp/gcp_test.go b/internal/bootstrap/gcp/gcp_test.go index d0d6ff15..073ef283 100644 --- a/internal/bootstrap/gcp/gcp_test.go +++ b/internal/bootstrap/gcp/gcp_test.go @@ -1235,9 +1235,74 @@ var _ = Describe("GCP Bootstrapper", func() { err := bs.EnsureHostsConfigured() Expect(err).NotTo(HaveOccurred()) }) + + It("creates the directory the installer uploads the age key to on every node", func() { + mkdirs := map[string]int{} + nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything). + RunAndReturn(func(n *node.Node, _ string, command string) error { + if command == "mkdir -p /etc/codesphere/secrets" { + mkdirs[n.GetName()]++ + } + return nil + }) + + Expect(bs.EnsureHostsConfigured()).To(Succeed()) + // The postgres node plus every cluster node of every data center. + Expect(mkdirs).To(Equal(map[string]int{ + "postgres": 1, + "k0s-1": 1, "k0s-2": 1, "k0s-3": 1, + "ceph-1": 1, "ceph-2": 1, "ceph-3": 1, + })) + }) + + It("creates a secondary data center's own secrets directory on its nodes only", func() { + secondary := &gcp.DataCenter{ID: 2, Suffix: "-dc2", SecretsDir: "/etc/codesphere/secrets-dc2"} + secondary.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1-dc2", nodeClient)} + secondary.CephNodes = []*node.Node{fakeNode("ceph-1-dc2", nodeClient)} + + bs.Env.DataCenters = []*gcp.DataCenter{ + { + ID: 1, + SecretsDir: "/etc/codesphere/secrets", + ControlPlaneNodes: bs.Env.ControlPlaneNodes, + CephNodes: bs.Env.CephNodes, + }, + secondary, + } + + mkdirs := map[string][]string{} + nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything). + RunAndReturn(func(n *node.Node, _ string, command string) error { + if dir, found := strings.CutPrefix(command, "mkdir -p "); found { + mkdirs[n.GetName()] = append(mkdirs[n.GetName()], dir) + } + return nil + }) + + Expect(bs.EnsureHostsConfigured()).To(Succeed()) + // Both paths on the secondary's nodes, because the installer's fixed path is + // created everywhere; the secondary's path on nobody else's. + Expect(mkdirs["k0s-1-dc2"]).To(ConsistOf("/etc/codesphere/secrets", "/etc/codesphere/secrets-dc2")) + Expect(mkdirs["ceph-1-dc2"]).To(ConsistOf("/etc/codesphere/secrets", "/etc/codesphere/secrets-dc2")) + Expect(mkdirs["k0s-1"]).To(ConsistOf("/etc/codesphere/secrets")) + Expect(mkdirs["postgres"]).To(ConsistOf("/etc/codesphere/secrets")) + }) }) Describe("Invalid cases", func() { + It("fails when the secrets directory cannot be created", func() { + nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything). + RunAndReturn(func(_ *node.Node, _ string, command string) error { + if command == "mkdir -p /etc/codesphere/secrets" { + return fmt.Errorf("ouch") + } + return nil + }) + + err := bs.EnsureHostsConfigured() + Expect(err).To(MatchError(ContainSubstring("failed to create secrets directory on postgres"))) + }) + It("fails when ConfigureInotifyWatches fails", func() { nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything).Return(fmt.Errorf("ouch"))