diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index e8ef40ce..72aaa895 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "log" - "path/filepath" "slices" "strings" "time" @@ -36,6 +35,24 @@ const ( RegistryTypeGitHub RegistryType = "github" ) +// remoteK0sConfigScriptPath is where each data center's k0s configuration script is placed on +// that data center's first control plane node. Data centers have separate nodes, so the path can +// be the same for all of them. +const remoteK0sConfigScriptPath = "/root/configure-k0s.sh" + +// installerNodeSecretsDir is where the Codesphere installer uploads a data center's age key on +// every one of that data center's nodes. The path is fixed even though the installer reads the key +// from the data center's own secrets.baseDir on the jumpbox, and the installer only creates +// baseDir on the node — so for a data center whose baseDir differs, the upload target would not +// exist. Creating it up front is harmless: data centers have separate nodes, so a node only ever +// holds its own data center's key. +const installerNodeSecretsDir = "/etc/codesphere/secrets" + +// vpcSubnetCIDR is the range of the project's single subnet, shared by all data centers. It is +// also each data center's ceph.nodesSubnet; their Ceph clusters stay separate because each has +// its own hosts, monitors and FSID. +const vpcSubnetCIDR = "10.10.0.0/20" + // CheckOMSManagedLabel checks if the given labels map indicates an OMS-managed project. // A project is considered OMS-managed if it has the 'oms-managed' label set to "true". func CheckOMSManagedLabel(labels map[string]string) bool { @@ -107,6 +124,30 @@ func (b *GCPBootstrapper) primaryDC() *DataCenter { return b.Env.DataCenters[0] } +// allNodes returns every node of the project: the jumpbox, the shared postgres node and all +// data centers' Ceph and k0s nodes. +func (b *GCPBootstrapper) allNodes() []*node.Node { + nodes := []*node.Node{b.Env.Jumpbox, b.Env.PostgreSQLNode} + for _, dc := range b.Env.DataCenters { + nodes = append(nodes, dc.ControlPlaneNodes...) + nodes = append(nodes, dc.CephNodes...) + } + + return nodes +} + +// clusterNodes returns every Ceph and k0s node of all data centers, i.e. all nodes except the +// jumpbox and the shared postgres node. +func (b *GCPBootstrapper) clusterNodes() []*node.Node { + nodes := []*node.Node{} + for _, dc := range b.Env.DataCenters { + nodes = append(nodes, dc.ControlPlaneNodes...) + nodes = append(nodes, dc.CephNodes...) + } + + return nodes +} + type CodesphereEnvironment struct { ProjectID string `json:"project_id"` ProjectTTL string `json:"project_ttl"` @@ -373,12 +414,14 @@ func (b *GCPBootstrapper) Bootstrap() error { } if b.Env.InstallVersion != "" || b.Env.InstallLocal != "" { - err = b.stlog.Step("Install Codesphere", b.InstallCodesphere) + err = b.InstallCodesphere() if err != nil { return fmt.Errorf("failed to install Codesphere: %w", err) } - err = b.stlog.Step("Run k0s config script", b.RunK0sConfigScript) + // Every data center is installed before any k0s script runs, so a script never patches + // gateway services an install has yet to create. + err = b.RunK0sConfigScript() if err != nil { return fmt.Errorf("failed to run k0s config script: %w", err) } @@ -393,8 +436,11 @@ func (b *GCPBootstrapper) Bootstrap() error { return nil } -// createTestUser creates a test user in the PostgreSQL instance using the testuser package and logs the credentials. +// createTestUser creates a test user in the shared PostgreSQL instance using the testuser package +// and logs the credentials. The user's team is homed in the primary data center. func (b *GCPBootstrapper) createTestUser() error { + b.ensureDataCenters() + if b.Env.PostgreSQLNode == nil { return fmt.Errorf("postgres node not found in bootstrap environment") } @@ -404,10 +450,11 @@ func (b *GCPBootstrapper) createTestUser() error { return fmt.Errorf("postgres node has no external IP") } - if b.Env.InstallConfig == nil { + primary := b.primaryDC() + if primary.InstallConfig == nil { return fmt.Errorf("install config not found in bootstrap environment") } - pgPasswordSecret := b.icg.GetVault().GetSecret(files.SecretPostgresPassword) + pgPasswordSecret := primary.ConfigManager().GetVault().GetSecret(files.SecretPostgresPassword) if pgPasswordSecret == nil || pgPasswordSecret.Fields == nil { return fmt.Errorf("postgres admin password not found in vault") } @@ -420,7 +467,7 @@ func (b *GCPBootstrapper) createTestUser() error { Password: pgPassword, DBName: testuser.DefaultDBName, SSLMode: "require", - DatacenterID: b.Env.DatacenterID, + DatacenterID: primary.ID, }) if err != nil { return err @@ -707,7 +754,7 @@ func (b *GCPBootstrapper) EnsureFirewallRules() error { Allowed: []*computepb.Allowed{ {IPProtocol: protoString("all")}, }, - SourceRanges: []string{"10.10.0.0/20"}, + SourceRanges: []string{vpcSubnetCIDR}, Description: protoString("Allow all internal traffic"), } err = b.GCPClient.CreateFirewallRule(b.Env.ProjectID, internalRule) @@ -770,19 +817,34 @@ func (b *GCPBootstrapper) EnsureFirewallRules() error { return nil } -// EnsureGatewayIPAddresses reserves the static external IP addresses for the ingress -// controllers of the cluster (gateway and public gateway) and the SSH workspace proxy. +// EnsureGatewayIPAddresses reserves the static external IP addresses of every data center: the +// ingress controllers of its cluster (gateway and public gateway) and its SSH workspace proxy. func (b *GCPBootstrapper) EnsureGatewayIPAddresses() error { + b.ensureDataCenters() + + for _, dc := range b.Env.DataCenters { + if err := b.ensureGatewayIPAddresses(dc); err != nil { + return err + } + } + b.mirrorPrimaryDataCenter() + + return nil +} + +// ensureGatewayIPAddresses reserves one data center's static external IP addresses. Their names +// carry the data-center suffix, so the primary data center keeps the unsuffixed names. +func (b *GCPBootstrapper) ensureGatewayIPAddresses(dc *DataCenter) error { var err error - b.Env.GatewayIP, err = b.EnsureExternalIP("gateway") + dc.GatewayIP, err = b.EnsureExternalIP("gateway" + dc.Suffix) if err != nil { return fmt.Errorf("failed to ensure gateway IP: %w", err) } - b.Env.PublicGatewayIP, err = b.EnsureExternalIP("public-gateway") + dc.PublicGatewayIP, err = b.EnsureExternalIP("public-gateway" + dc.Suffix) if err != nil { return fmt.Errorf("failed to ensure public gateway IP: %w", err) } - b.Env.SshProxyIP, err = b.EnsureExternalIP("ssh-proxy") + dc.SshProxyIP, err = b.EnsureExternalIP("ssh-proxy" + dc.Suffix) if err != nil { return fmt.Errorf("failed to ensure ssh proxy IP: %w", err) } @@ -823,14 +885,9 @@ func (b *GCPBootstrapper) EnsureExternalIP(name string) (string, error) { } func (b *GCPBootstrapper) EnsureRootLoginEnabled() error { - allNodes := []*node.Node{ - b.Env.Jumpbox, - } - allNodes = append(allNodes, b.Env.ControlPlaneNodes...) - allNodes = append(allNodes, b.Env.PostgreSQLNode) - allNodes = append(allNodes, b.Env.CephNodes...) + b.ensureDataCenters() - for _, node := range allNodes { + for _, node := range b.allNodes() { err := b.stlog.Substep(fmt.Sprintf("Ensuring root login enabled on %s", node.GetName()), func() error { return b.ensureRootLoginEnabledInNode(node) }) @@ -916,8 +973,9 @@ func (b *GCPBootstrapper) EnsureOmsInstalled() (err error) { } func (b *GCPBootstrapper) EnsureHostsConfigured() error { - allNodes := append(b.Env.ControlPlaneNodes, b.Env.PostgreSQLNode) - allNodes = append(allNodes, b.Env.CephNodes...) + b.ensureDataCenters() + + allNodes := append([]*node.Node{b.Env.PostgreSQLNode}, b.clusterNodes()...) for _, node := range allNodes { if !node.HasInotifyWatchesConfigured() { @@ -932,6 +990,25 @@ func (b *GCPBootstrapper) EnsureHostsConfigured() error { return fmt.Errorf("failed to configure memory map on %s: %w", node.GetName(), err) } } + err := node.RunSSHCommand("root", "mkdir -p "+installerNodeSecretsDir) + if err != nil { + return fmt.Errorf("failed to create secrets directory on %s: %w", node.GetName(), err) + } + } + + // A secondary data center's secrets directory differs from the fixed path above, so create that + // one too on its own nodes. Nodes belong to exactly one data center, so no node gets a foreign + // data center's directory. + for _, dc := range b.Env.DataCenters { + if dc.SecretsDir == installerNodeSecretsDir { + continue + } + for _, n := range append(append([]*node.Node{}, dc.ControlPlaneNodes...), dc.CephNodes...) { + err := n.RunSSHCommand("root", "mkdir -p "+dc.SecretsDir) + if err != nil { + return fmt.Errorf("failed to create secrets directory on %s: %w", n.GetName(), err) + } + } } return nil @@ -1079,15 +1156,42 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error { return nil } +// InstallCodesphere installs Codesphere into every data center from the shared jumpbox, in +// ascending data center order. The order matters: the primary data center's install creates the +// database, roles and schema that the secondary ones reuse. func (b *GCPBootstrapper) InstallCodesphere() error { + b.ensureDataCenters() + fullPackageFilename, err := b.ensureCodespherePackageOnJumpbox() if err != nil { return fmt.Errorf("failed to ensure Codesphere package on jumpbox: %w", err) } - err = b.runInstallCommand(fullPackageFilename) - if err != nil { - return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err) + for _, dc := range b.Env.DataCenters { + err = b.stlog.Step(dc.StepName("Install Codesphere"), func() error { + return b.runInstallCommand(dc, fullPackageFilename) + }) + if err != nil { + return fmt.Errorf("failed to install Codesphere from jumpbox (data center %d): %w", dc.ID, err) + } + } + + return nil +} + +// RunK0sConfigScript runs every data center's k0s configuration script on its first control +// plane node. It requires that data center's Codesphere install to have completed, since the +// script patches the gateway services the install creates. +func (b *GCPBootstrapper) RunK0sConfigScript() error { + b.ensureDataCenters() + + for _, dc := range b.Env.DataCenters { + err := b.stlog.Step(dc.StepName("Run k0s config script"), func() error { + return b.runK0sConfigScript(dc) + }) + if err != nil { + return err + } } return nil @@ -1128,11 +1232,16 @@ func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() (string, error) { return fullPackageFilename, nil } -func (b *GCPBootstrapper) runInstallCommand(packageFilename string) error { - b.stlog.Logf("Installing Codesphere...") - installCmd := fmt.Sprintf("oms install codesphere -c /etc/codesphere/config.yaml -k %s/age_key.txt --vault %s -p %s%s", - b.Env.SecretsDir, filepath.Join(b.Env.SecretsDir, "prod.vault.yaml"), packageFilename, b.generateSkipStepsArg()) - return b.Env.Jumpbox.RunSSHCommand("root", installCmd) +func (b *GCPBootstrapper) runInstallCommand(dc *DataCenter, packageFilename string) error { + b.stlog.Logf("Installing Codesphere in data center %d...", dc.ID) + return b.Env.Jumpbox.RunSSHCommand("root", b.InstallCommand(dc, packageFilename)) +} + +// InstallCommand returns the command that installs Codesphere into the given data center from +// the jumpbox. It is also printed for the operator when the bootstrap does not install itself. +func (b *GCPBootstrapper) InstallCommand(dc *DataCenter, packageFilename string) string { + return fmt.Sprintf("oms install codesphere -c %s -k %s --vault %s -p %s%s", + dc.RemoteConfigPath, dc.RemoteAgeKeyPath(), dc.RemoteVaultPath(), packageFilename, b.generateSkipStepsArg()) } func (b *GCPBootstrapper) generateSkipStepsArg() string { @@ -1147,7 +1256,21 @@ func (b *GCPBootstrapper) generateSkipStepsArg() string { return " -s " + strings.Join(skipSteps, ",") } +// GenerateK0sConfigScript writes and uploads the k0s cloud-provider configuration script of +// every data center to that data center's first control plane node. func (b *GCPBootstrapper) GenerateK0sConfigScript() error { + b.ensureDataCenters() + + for _, dc := range b.Env.DataCenters { + if err := b.generateK0sConfigScript(dc); err != nil { + return err + } + } + + return nil +} + +func (b *GCPBootstrapper) generateK0sConfigScript(dc *DataCenter) error { script := `#!/bin/bash cat < cloud.conf @@ -1215,14 +1338,14 @@ $KUBECTL apply -f https://raw.githubusercontent.com/kubernetes/cloud-provider-gc $KUBECTL apply -f cc-deployment.yaml # set loadBalancerIP for public-gateway-controller and gateway-controller -$KUBECTL patch svc public-gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + b.Env.PublicGatewayIP + `'"}}' -$KUBECTL patch svc gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + b.Env.GatewayIP + `'"}}' +$KUBECTL patch svc public-gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + dc.PublicGatewayIP + `'"}}' +$KUBECTL patch svc gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + dc.GatewayIP + `'"}}' sed -i 's/k0scontroller/k0scontroller --enable-cloud-provider/g' /etc/systemd/system/k0scontroller.service -ssh -o StrictHostKeyChecking=no root@` + b.Env.ControlPlaneNodes[1].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" +ssh -o StrictHostKeyChecking=no root@` + dc.ControlPlaneNodes[1].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" -ssh -o StrictHostKeyChecking=no root@` + b.Env.ControlPlaneNodes[2].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" +ssh -o StrictHostKeyChecking=no root@` + dc.ControlPlaneNodes[2].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker" systemctl daemon-reload systemctl restart k0scontroller @@ -1231,25 +1354,27 @@ systemctl restart k0scontroller // --enable-cloud-provider on worker nodes systemd file /etc/systemd/system/k0sworker.service // in addition on the first node: /etc/systemd/system/k0scontroller.service the flag --enable-cloud-provider - err := b.fw.WriteFile("configure-k0s.sh", []byte(script), 0755) + localScript := dc.K0sConfigScriptPath() + err := b.fw.WriteFile(localScript, []byte(script), 0755) if err != nil { - return fmt.Errorf("failed to write configure-k0s.sh: %w", err) + return fmt.Errorf("failed to write %s: %w", localScript, err) } - err = b.Env.ControlPlaneNodes[0].NodeClient.CopyFile(b.Env.ControlPlaneNodes[0], "configure-k0s.sh", "/root/configure-k0s.sh") + controller := dc.ControlPlaneNodes[0] + err = controller.NodeClient.CopyFile(controller, localScript, remoteK0sConfigScriptPath) if err != nil { - return fmt.Errorf("failed to copy configure-k0s.sh to control plane node: %w", err) + return fmt.Errorf("failed to copy %s to control plane node: %w", localScript, err) } - err = b.Env.ControlPlaneNodes[0].RunSSHCommand("root", "chmod +x /root/configure-k0s.sh") + err = controller.RunSSHCommand("root", "chmod +x "+remoteK0sConfigScriptPath) if err != nil { - return fmt.Errorf("failed to make configure-k0s.sh executable on control plane node: %w", err) + return fmt.Errorf("failed to make %s executable: %w", localScript, err) } return nil } -func (b *GCPBootstrapper) RunK0sConfigScript() error { - err := b.Env.ControlPlaneNodes[0].RunSSHCommand("root", "/root/configure-k0s.sh") +func (b *GCPBootstrapper) runK0sConfigScript(dc *DataCenter) error { + err := dc.ControlPlaneNodes[0].RunSSHCommand("root", remoteK0sConfigScriptPath) if err != nil { - return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err) + return fmt.Errorf("failed to configure k0s in data center %d: %w", dc.ID, err) } return nil diff --git a/internal/bootstrap/gcp/gcp_test.go b/internal/bootstrap/gcp/gcp_test.go index d0d6ff15..073ef283 100644 --- a/internal/bootstrap/gcp/gcp_test.go +++ b/internal/bootstrap/gcp/gcp_test.go @@ -1235,9 +1235,74 @@ var _ = Describe("GCP Bootstrapper", func() { err := bs.EnsureHostsConfigured() Expect(err).NotTo(HaveOccurred()) }) + + It("creates the directory the installer uploads the age key to on every node", func() { + mkdirs := map[string]int{} + nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything). + RunAndReturn(func(n *node.Node, _ string, command string) error { + if command == "mkdir -p /etc/codesphere/secrets" { + mkdirs[n.GetName()]++ + } + return nil + }) + + Expect(bs.EnsureHostsConfigured()).To(Succeed()) + // The postgres node plus every cluster node of every data center. + Expect(mkdirs).To(Equal(map[string]int{ + "postgres": 1, + "k0s-1": 1, "k0s-2": 1, "k0s-3": 1, + "ceph-1": 1, "ceph-2": 1, "ceph-3": 1, + })) + }) + + It("creates a secondary data center's own secrets directory on its nodes only", func() { + secondary := &gcp.DataCenter{ID: 2, Suffix: "-dc2", SecretsDir: "/etc/codesphere/secrets-dc2"} + secondary.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1-dc2", nodeClient)} + secondary.CephNodes = []*node.Node{fakeNode("ceph-1-dc2", nodeClient)} + + bs.Env.DataCenters = []*gcp.DataCenter{ + { + ID: 1, + SecretsDir: "/etc/codesphere/secrets", + ControlPlaneNodes: bs.Env.ControlPlaneNodes, + CephNodes: bs.Env.CephNodes, + }, + secondary, + } + + mkdirs := map[string][]string{} + nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything). + RunAndReturn(func(n *node.Node, _ string, command string) error { + if dir, found := strings.CutPrefix(command, "mkdir -p "); found { + mkdirs[n.GetName()] = append(mkdirs[n.GetName()], dir) + } + return nil + }) + + Expect(bs.EnsureHostsConfigured()).To(Succeed()) + // Both paths on the secondary's nodes, because the installer's fixed path is + // created everywhere; the secondary's path on nobody else's. + Expect(mkdirs["k0s-1-dc2"]).To(ConsistOf("/etc/codesphere/secrets", "/etc/codesphere/secrets-dc2")) + Expect(mkdirs["ceph-1-dc2"]).To(ConsistOf("/etc/codesphere/secrets", "/etc/codesphere/secrets-dc2")) + Expect(mkdirs["k0s-1"]).To(ConsistOf("/etc/codesphere/secrets")) + Expect(mkdirs["postgres"]).To(ConsistOf("/etc/codesphere/secrets")) + }) }) Describe("Invalid cases", func() { + It("fails when the secrets directory cannot be created", func() { + nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything). + RunAndReturn(func(_ *node.Node, _ string, command string) error { + if command == "mkdir -p /etc/codesphere/secrets" { + return fmt.Errorf("ouch") + } + return nil + }) + + err := bs.EnsureHostsConfigured() + Expect(err).To(MatchError(ContainSubstring("failed to create secrets directory on postgres"))) + }) + It("fails when ConfigureInotifyWatches fails", func() { nodeClient.EXPECT().RunCommand(mock.Anything, "root", mock.Anything).Return(fmt.Errorf("ouch"))