Skip to content

feat(gcp): configure hosts, gateways and installs per data center #627

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
NJona wants to merge 1 commit into from
+233 −43
Open

feat(gcp): configure hosts, gateways and installs per data center #627

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
211 changes: 168 additions & 43 deletions internal/bootstrap/gcp/gcp.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"errors"
"fmt"
"log"
"path/filepath"
"slices"
"strings"
"time"
Expand Down Expand Up @@ -36,6 +35,24 @@ const (
RegistryTypeGitHub RegistryType = "github"
)

// remoteK0sConfigScriptPath is where each data center's k0s configuration script is placed on
// that data center's first control plane node. Data centers have separate nodes, so the path can
// be the same for all of them.
const remoteK0sConfigScriptPath = "/root/configure-k0s.sh"

// installerNodeSecretsDir is where the Codesphere installer uploads a data center's age key on
// every one of that data center's nodes. The path is fixed even though the installer reads the key
// from the data center's own secrets.baseDir on the jumpbox, and the installer only creates
// baseDir on the node — so for a data center whose baseDir differs, the upload target would not
// exist. Creating it up front is harmless: data centers have separate nodes, so a node only ever
// holds its own data center's key.
const installerNodeSecretsDir = "/etc/codesphere/secrets"

// vpcSubnetCIDR is the range of the project's single subnet, shared by all data centers. It is
// also each data center's ceph.nodesSubnet; their Ceph clusters stay separate because each has
// its own hosts, monitors and FSID.
const vpcSubnetCIDR = "10.10.0.0/20"

// CheckOMSManagedLabel checks if the given labels map indicates an OMS-managed project.
// A project is considered OMS-managed if it has the 'oms-managed' label set to "true".
func CheckOMSManagedLabel(labels map[string]string) bool {
Expand Down Expand Up @@ -107,6 +124,30 @@ func (b *GCPBootstrapper) primaryDC() *DataCenter {
return b.Env.DataCenters[0]
}

// allNodes returns every node of the project: the jumpbox, the shared postgres node and all
// data centers' Ceph and k0s nodes.
func (b *GCPBootstrapper) allNodes() []*node.Node {
nodes := []*node.Node{b.Env.Jumpbox, b.Env.PostgreSQLNode}
for _, dc := range b.Env.DataCenters {
nodes = append(nodes, dc.ControlPlaneNodes...)
nodes = append(nodes, dc.CephNodes...)
}

return nodes
}

// clusterNodes returns every Ceph and k0s node of all data centers, i.e. all nodes except the
// jumpbox and the shared postgres node.
func (b *GCPBootstrapper) clusterNodes() []*node.Node {
nodes := []*node.Node{}
for _, dc := range b.Env.DataCenters {
nodes = append(nodes, dc.ControlPlaneNodes...)
nodes = append(nodes, dc.CephNodes...)
}

return nodes
}

type CodesphereEnvironment struct {
ProjectID string `json:"project_id"`
ProjectTTL string `json:"project_ttl"`
Expand Down Expand Up @@ -373,12 +414,14 @@ func (b *GCPBootstrapper) Bootstrap() error {
}

if b.Env.InstallVersion != "" || b.Env.InstallLocal != "" {
err = b.stlog.Step("Install Codesphere", b.InstallCodesphere)
err = b.InstallCodesphere()
if err != nil {
return fmt.Errorf("failed to install Codesphere: %w", err)
}

err = b.stlog.Step("Run k0s config script", b.RunK0sConfigScript)
// Every data center is installed before any k0s script runs, so a script never patches
// gateway services an install has yet to create.
err = b.RunK0sConfigScript()
if err != nil {
return fmt.Errorf("failed to run k0s config script: %w", err)
}
Expand All @@ -393,8 +436,11 @@ func (b *GCPBootstrapper) Bootstrap() error {
return nil
}

// createTestUser creates a test user in the PostgreSQL instance using the testuser package and logs the credentials.
// createTestUser creates a test user in the shared PostgreSQL instance using the testuser package
// and logs the credentials. The user's team is homed in the primary data center.
func (b *GCPBootstrapper) createTestUser() error {
b.ensureDataCenters()

if b.Env.PostgreSQLNode == nil {
return fmt.Errorf("postgres node not found in bootstrap environment")
}
Expand All @@ -404,10 +450,11 @@ func (b *GCPBootstrapper) createTestUser() error {
return fmt.Errorf("postgres node has no external IP")
}

if b.Env.InstallConfig == nil {
primary := b.primaryDC()
if primary.InstallConfig == nil {
return fmt.Errorf("install config not found in bootstrap environment")
}
pgPasswordSecret := b.icg.GetVault().GetSecret(files.SecretPostgresPassword)
pgPasswordSecret := primary.ConfigManager().GetVault().GetSecret(files.SecretPostgresPassword)
if pgPasswordSecret == nil || pgPasswordSecret.Fields == nil {
return fmt.Errorf("postgres admin password not found in vault")
}
Expand All @@ -420,7 +467,7 @@ func (b *GCPBootstrapper) createTestUser() error {
Password: pgPassword,
DBName: testuser.DefaultDBName,
SSLMode: "require",
DatacenterID: b.Env.DatacenterID,
DatacenterID: primary.ID,
})
if err != nil {
return err
Expand Down Expand Up @@ -707,7 +754,7 @@ func (b *GCPBootstrapper) EnsureFirewallRules() error {
Allowed: []*computepb.Allowed{
{IPProtocol: protoString("all")},
},
SourceRanges: []string{"10.10.0.0/20"},
SourceRanges: []string{vpcSubnetCIDR},
Description: protoString("Allow all internal traffic"),
}
err = b.GCPClient.CreateFirewallRule(b.Env.ProjectID, internalRule)
Expand Down Expand Up @@ -770,19 +817,34 @@ func (b *GCPBootstrapper) EnsureFirewallRules() error {
return nil
}

// EnsureGatewayIPAddresses reserves the static external IP addresses for the ingress
// controllers of the cluster (gateway and public gateway) and the SSH workspace proxy.
// EnsureGatewayIPAddresses reserves the static external IP addresses of every data center: the
// ingress controllers of its cluster (gateway and public gateway) and its SSH workspace proxy.
func (b *GCPBootstrapper) EnsureGatewayIPAddresses() error {
b.ensureDataCenters()

for _, dc := range b.Env.DataCenters {
if err := b.ensureGatewayIPAddresses(dc); err != nil {
return err
}
}
b.mirrorPrimaryDataCenter()

return nil
}

// ensureGatewayIPAddresses reserves one data center's static external IP addresses. Their names
// carry the data-center suffix, so the primary data center keeps the unsuffixed names.
func (b *GCPBootstrapper) ensureGatewayIPAddresses(dc *DataCenter) error {
var err error
b.Env.GatewayIP, err = b.EnsureExternalIP("gateway")
dc.GatewayIP, err = b.EnsureExternalIP("gateway" + dc.Suffix)
if err != nil {
return fmt.Errorf("failed to ensure gateway IP: %w", err)
}
b.Env.PublicGatewayIP, err = b.EnsureExternalIP("public-gateway")
dc.PublicGatewayIP, err = b.EnsureExternalIP("public-gateway" + dc.Suffix)
if err != nil {
return fmt.Errorf("failed to ensure public gateway IP: %w", err)
}
b.Env.SshProxyIP, err = b.EnsureExternalIP("ssh-proxy")
dc.SshProxyIP, err = b.EnsureExternalIP("ssh-proxy" + dc.Suffix)
if err != nil {
return fmt.Errorf("failed to ensure ssh proxy IP: %w", err)
}
Expand Down Expand Up @@ -823,14 +885,9 @@ func (b *GCPBootstrapper) EnsureExternalIP(name string) (string, error) {
}

func (b *GCPBootstrapper) EnsureRootLoginEnabled() error {
allNodes := []*node.Node{
b.Env.Jumpbox,
}
allNodes = append(allNodes, b.Env.ControlPlaneNodes...)
allNodes = append(allNodes, b.Env.PostgreSQLNode)
allNodes = append(allNodes, b.Env.CephNodes...)
b.ensureDataCenters()

for _, node := range allNodes {
for _, node := range b.allNodes() {
err := b.stlog.Substep(fmt.Sprintf("Ensuring root login enabled on %s", node.GetName()), func() error {
return b.ensureRootLoginEnabledInNode(node)
})
Expand Down Expand Up @@ -916,8 +973,9 @@ func (b *GCPBootstrapper) EnsureOmsInstalled() (err error) {
}

func (b *GCPBootstrapper) EnsureHostsConfigured() error {
allNodes := append(b.Env.ControlPlaneNodes, b.Env.PostgreSQLNode)
allNodes = append(allNodes, b.Env.CephNodes...)
b.ensureDataCenters()

allNodes := append([]*node.Node{b.Env.PostgreSQLNode}, b.clusterNodes()...)

for _, node := range allNodes {
if !node.HasInotifyWatchesConfigured() {
Expand All @@ -932,6 +990,25 @@ func (b *GCPBootstrapper) EnsureHostsConfigured() error {
return fmt.Errorf("failed to configure memory map on %s: %w", node.GetName(), err)
}
}
err := node.RunSSHCommand("root", "mkdir -p "+installerNodeSecretsDir)
if err != nil {
return fmt.Errorf("failed to create secrets directory on %s: %w", node.GetName(), err)
}
}

// A secondary data center's secrets directory differs from the fixed path above, so create that
// one too on its own nodes. Nodes belong to exactly one data center, so no node gets a foreign
// data center's directory.
for _, dc := range b.Env.DataCenters {
if dc.SecretsDir == installerNodeSecretsDir {
continue
}
for _, n := range append(append([]*node.Node{}, dc.ControlPlaneNodes...), dc.CephNodes...) {
err := n.RunSSHCommand("root", "mkdir -p "+dc.SecretsDir)
if err != nil {
return fmt.Errorf("failed to create secrets directory on %s: %w", n.GetName(), err)
}
}
}

return nil
Expand Down Expand Up @@ -1079,15 +1156,42 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error {
return nil
}

// InstallCodesphere installs Codesphere into every data center from the shared jumpbox, in
// ascending data center order. The order matters: the primary data center's install creates the
// database, roles and schema that the secondary ones reuse.
func (b *GCPBootstrapper) InstallCodesphere() error {
b.ensureDataCenters()

fullPackageFilename, err := b.ensureCodespherePackageOnJumpbox()
if err != nil {
return fmt.Errorf("failed to ensure Codesphere package on jumpbox: %w", err)
}

err = b.runInstallCommand(fullPackageFilename)
if err != nil {
return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err)
for _, dc := range b.Env.DataCenters {
err = b.stlog.Step(dc.StepName("Install Codesphere"), func() error {
return b.runInstallCommand(dc, fullPackageFilename)
})
if err != nil {
return fmt.Errorf("failed to install Codesphere from jumpbox (data center %d): %w", dc.ID, err)
}
}

return nil
}

// RunK0sConfigScript runs every data center's k0s configuration script on its first control
// plane node. It requires that data center's Codesphere install to have completed, since the
// script patches the gateway services the install creates.
func (b *GCPBootstrapper) RunK0sConfigScript() error {
b.ensureDataCenters()

for _, dc := range b.Env.DataCenters {
err := b.stlog.Step(dc.StepName("Run k0s config script"), func() error {
return b.runK0sConfigScript(dc)
})
if err != nil {
return err
}
}

return nil
Expand Down Expand Up @@ -1128,11 +1232,16 @@ func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() (string, error) {
return fullPackageFilename, nil
}

func (b *GCPBootstrapper) runInstallCommand(packageFilename string) error {
b.stlog.Logf("Installing Codesphere...")
installCmd := fmt.Sprintf("oms install codesphere -c /etc/codesphere/config.yaml -k %s/age_key.txt --vault %s -p %s%s",
b.Env.SecretsDir, filepath.Join(b.Env.SecretsDir, "prod.vault.yaml"), packageFilename, b.generateSkipStepsArg())
return b.Env.Jumpbox.RunSSHCommand("root", installCmd)
func (b *GCPBootstrapper) runInstallCommand(dc *DataCenter, packageFilename string) error {
b.stlog.Logf("Installing Codesphere in data center %d...", dc.ID)
return b.Env.Jumpbox.RunSSHCommand("root", b.InstallCommand(dc, packageFilename))
}

// InstallCommand returns the command that installs Codesphere into the given data center from
// the jumpbox. It is also printed for the operator when the bootstrap does not install itself.
func (b *GCPBootstrapper) InstallCommand(dc *DataCenter, packageFilename string) string {
return fmt.Sprintf("oms install codesphere -c %s -k %s --vault %s -p %s%s",
dc.RemoteConfigPath, dc.RemoteAgeKeyPath(), dc.RemoteVaultPath(), packageFilename, b.generateSkipStepsArg())
}

func (b *GCPBootstrapper) generateSkipStepsArg() string {
Expand All @@ -1147,7 +1256,21 @@ func (b *GCPBootstrapper) generateSkipStepsArg() string {
return " -s " + strings.Join(skipSteps, ",")
}

// GenerateK0sConfigScript writes and uploads the k0s cloud-provider configuration script of
// every data center to that data center's first control plane node.
func (b *GCPBootstrapper) GenerateK0sConfigScript() error {
b.ensureDataCenters()

for _, dc := range b.Env.DataCenters {
if err := b.generateK0sConfigScript(dc); err != nil {
return err
}
}

return nil
}

func (b *GCPBootstrapper) generateK0sConfigScript(dc *DataCenter) error {
script := `#!/bin/bash

cat <<EOF > cloud.conf
Expand Down Expand Up @@ -1215,14 +1338,14 @@ $KUBECTL apply -f https://raw.githubusercontent.com/kubernetes/cloud-provider-gc
$KUBECTL apply -f cc-deployment.yaml

# set loadBalancerIP for public-gateway-controller and gateway-controller
$KUBECTL patch svc public-gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + b.Env.PublicGatewayIP + `'"}}'
$KUBECTL patch svc gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + b.Env.GatewayIP + `'"}}'
$KUBECTL patch svc public-gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + dc.PublicGatewayIP + `'"}}'
$KUBECTL patch svc gateway-controller -n codesphere -p '{"spec": {"loadBalancerIP": "'` + dc.GatewayIP + `'"}}'

sed -i 's/k0scontroller/k0scontroller --enable-cloud-provider/g' /etc/systemd/system/k0scontroller.service

ssh -o StrictHostKeyChecking=no root@` + b.Env.ControlPlaneNodes[1].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker"
ssh -o StrictHostKeyChecking=no root@` + dc.ControlPlaneNodes[1].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker"

ssh -o StrictHostKeyChecking=no root@` + b.Env.ControlPlaneNodes[2].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker"
ssh -o StrictHostKeyChecking=no root@` + dc.ControlPlaneNodes[2].GetInternalIP() + ` "sed -i 's/k0sworker/k0sworker --enable-cloud-provider/g' /etc/systemd/system/k0sworker.service; systemctl daemon-reload; systemctl restart k0sworker"

systemctl daemon-reload
systemctl restart k0scontroller
Expand All @@ -1231,25 +1354,27 @@ systemctl restart k0scontroller
// --enable-cloud-provider on worker nodes systemd file /etc/systemd/system/k0sworker.service
// in addition on the first node: /etc/systemd/system/k0scontroller.service the flag --enable-cloud-provider

err := b.fw.WriteFile("configure-k0s.sh", []byte(script), 0755)
localScript := dc.K0sConfigScriptPath()
err := b.fw.WriteFile(localScript, []byte(script), 0755)
if err != nil {
return fmt.Errorf("failed to write configure-k0s.sh: %w", err)
return fmt.Errorf("failed to write %s: %w", localScript, err)
}
err = b.Env.ControlPlaneNodes[0].NodeClient.CopyFile(b.Env.ControlPlaneNodes[0], "configure-k0s.sh", "/root/configure-k0s.sh")
controller := dc.ControlPlaneNodes[0]
err = controller.NodeClient.CopyFile(controller, localScript, remoteK0sConfigScriptPath)
if err != nil {
return fmt.Errorf("failed to copy configure-k0s.sh to control plane node: %w", err)
return fmt.Errorf("failed to copy %s to control plane node: %w", localScript, err)
}
err = b.Env.ControlPlaneNodes[0].RunSSHCommand("root", "chmod +x /root/configure-k0s.sh")
err = controller.RunSSHCommand("root", "chmod +x "+remoteK0sConfigScriptPath)
if err != nil {
return fmt.Errorf("failed to make configure-k0s.sh executable on control plane node: %w", err)
return fmt.Errorf("failed to make %s executable: %w", localScript, err)
}
return nil
}

func (b *GCPBootstrapper) RunK0sConfigScript() error {
err := b.Env.ControlPlaneNodes[0].RunSSHCommand("root", "/root/configure-k0s.sh")
func (b *GCPBootstrapper) runK0sConfigScript(dc *DataCenter) error {
err := dc.ControlPlaneNodes[0].RunSSHCommand("root", remoteK0sConfigScriptPath)
if err != nil {
return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err)
return fmt.Errorf("failed to configure k0s in data center %d: %w", dc.ID, err)
}

return nil
Expand Down
Loading
Loading