From d92a5fa51f123318ddd0ce21d2a1e8dbbbfa5d2d Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 15:00:29 +0200 Subject: [PATCH] feat(gcp): derive the VM definitions per data center MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The VM list was a package-level global describing one data center. It becomes a function of the environment: the project-shared jumpbox and postgres VMs plus each data center's three Ceph and three k0s nodes, whose names carry that data center's suffix. A single-DC bootstrap therefore still gets exactly ceph-1..3 and k0s-1..3. EnsureComputeInstances routes each instance into its own data center by the definition's DataCenterID instead of inferring placement from the VM's tag, and sorts each data center's nodes independently — the install config assigns roles by index. restart-vms resolves the valid VM names from the infra file's data center layout, so a node of a second data center can be restarted by name. It now also reads the infra file when --project-id and --zone are given, best-effort, since only the file knows the layout. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- cli/cmd/bootstrap_gcp_restart_vms.go | 53 ++++--- docs/oms_beta_bootstrap-gcp_restart-vms.md | 3 + internal/bootstrap/gcp/gce.go | 154 +++++++++++++++----- internal/bootstrap/gcp/gce_test.go | 55 +++++++ internal/bootstrap/gcp/test_helpers_test.go | 9 ++ 5 files changed, 213 insertions(+), 61 deletions(-) diff --git a/cli/cmd/bootstrap_gcp_restart_vms.go b/cli/cmd/bootstrap_gcp_restart_vms.go index 900b9353..a35ffebb 100644 --- a/cli/cmd/bootstrap_gcp_restart_vms.go +++ b/cli/cmd/bootstrap_gcp_restart_vms.go @@ -28,32 +28,46 @@ type BootstrapGcpRestartVMsOpts struct { Name string } -// resolveProjectAndZone returns the project ID and zone from flags or the infra file. -// If both flags are set they are used directly; if neither is set, the infra file is read. -// Providing only one of --project-id / --zone is an error. -func (c *BootstrapGcpRestartVMsCmd) resolveProjectAndZone(fw intutil.FileIO) (string, string, error) { +// resolveEnvironment returns the environment to restart VMs in. Project ID and zone come from +// the flags or, when neither is set, from the infra file. Providing only one of +// --project-id / --zone is an error. +// +// The data center layout always comes from the infra file, since it determines the VM names. It +// is read best-effort when the flags supply project and zone, in which case a missing file just +// means single-data-center names. +func (c *BootstrapGcpRestartVMsCmd) resolveEnvironment(fw intutil.FileIO) (*gcp.CodesphereEnvironment, error) { projectID := c.Opts.ProjectID zone := c.Opts.Zone if (projectID == "") != (zone == "") { - return "", "", fmt.Errorf("--project-id and --zone must be provided together") - } - if projectID != "" { - return projectID, zone, nil + return nil, fmt.Errorf("--project-id and --zone must be provided together") } infraFilePath := gcp.GetInfraFilePath() infraEnv, exists, err := gcp.LoadInfraFile(fw, infraFilePath) if err != nil { - return "", "", fmt.Errorf("failed to load infra file: %w", err) - } - if !exists { - return "", "", fmt.Errorf("infra file not found at %s; use --project-id and --zone flags", infraFilePath) + if projectID == "" { + return nil, fmt.Errorf("failed to load infra file: %w", err) + } + log.Printf("Warning: %v", err) } - if infraEnv.ProjectID == "" || infraEnv.Zone == "" { - return "", "", fmt.Errorf("infra file is missing project ID or zone; use --project-id and --zone flags") + + if projectID == "" { + if !exists { + return nil, fmt.Errorf("infra file not found at %s; use --project-id and --zone flags", infraFilePath) + } + if infraEnv.ProjectID == "" || infraEnv.Zone == "" { + return nil, fmt.Errorf("infra file is missing project ID or zone; use --project-id and --zone flags") + } + projectID, zone = infraEnv.ProjectID, infraEnv.Zone } - return infraEnv.ProjectID, infraEnv.Zone, nil + + return &gcp.CodesphereEnvironment{ + ProjectID: projectID, + Zone: zone, + MultiDC: infraEnv.MultiDC, + DataCenters: infraEnv.DataCenters, + }, nil } func (c *BootstrapGcpRestartVMsCmd) RunE(_ *cobra.Command, _ []string) error { @@ -61,18 +75,14 @@ func (c *BootstrapGcpRestartVMsCmd) RunE(_ *cobra.Command, _ []string) error { stlog := bootstrap.NewStepLogger(false) fw := intutil.NewFilesystemWriter() - projectID, zone, err := c.resolveProjectAndZone(fw) + csEnv, err := c.resolveEnvironment(fw) if err != nil { return err } + projectID, zone := csEnv.ProjectID, csEnv.Zone gcpClient := gcp.NewGCPClient(ctx, stlog, os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")) - csEnv := &gcp.CodesphereEnvironment{ - ProjectID: projectID, - Zone: zone, - } - bs, err := gcp.NewGCPBootstrapper( ctx, nil, stlog, csEnv, nil, gcpClient, fw, nil, nil, intutil.NewTime(), nil, @@ -112,6 +122,7 @@ func AddBootstrapGcpRestartVMsCmd(bootstrapGcp *cobra.Command, opts *util.Global {Desc: "Restart all VMs using project info from the local infra file"}, {Cmd: "--name jumpbox", Desc: "Restart only the jumpbox VM"}, {Cmd: "--name k0s-1", Desc: "Restart a specific k0s node"}, + {Cmd: "--name k0s-1-dc2", Desc: "Restart a node of the second data center of a --multi-dc bootstrap"}, {Cmd: "--project-id my-project --zone us-central1-a", Desc: "Restart all VMs with explicit project and zone"}, {Cmd: "--project-id my-project --zone us-central1-a --name ceph-1", Desc: "Restart a specific VM with explicit project and zone"}, }), diff --git a/docs/oms_beta_bootstrap-gcp_restart-vms.md b/docs/oms_beta_bootstrap-gcp_restart-vms.md index 9eeb152e..0f723624 100644 --- a/docs/oms_beta_bootstrap-gcp_restart-vms.md +++ b/docs/oms_beta_bootstrap-gcp_restart-vms.md @@ -26,6 +26,9 @@ $ oms beta bootstrap-gcp restart-vms --name jumpbox # Restart a specific k0s node $ oms beta bootstrap-gcp restart-vms --name k0s-1 +# Restart a node of the second data center of a --multi-dc bootstrap +$ oms beta bootstrap-gcp restart-vms --name k0s-1-dc2 + # Restart all VMs with explicit project and zone $ oms beta bootstrap-gcp restart-vms --project-id my-project --zone us-central1-a diff --git a/internal/bootstrap/gcp/gce.go b/internal/bootstrap/gcp/gce.go index a7b7fc66..10446080 100644 --- a/internal/bootstrap/gcp/gce.go +++ b/internal/bootstrap/gcp/gce.go @@ -13,6 +13,7 @@ import ( "time" "cloud.google.com/go/compute/apiv1/computepb" + "github.com/codesphere-cloud/oms/internal/bootstrap/datacenter" "github.com/codesphere-cloud/oms/internal/github" "github.com/codesphere-cloud/oms/internal/installer/node" "github.com/codesphere-cloud/oms/internal/util" @@ -24,18 +25,69 @@ type VMDef struct { Tags []string AdditionalDisks []int64 ExternalIP bool + // DataCenterID is the data center the VM belongs to, or 0 for the project-shared VMs + // (jumpbox and postgres) that every data center uses. + DataCenterID int } -// Example VM definitions (expand as needed) -var vmDefs = []VMDef{ - {"jumpbox", "e2-medium", []string{"jumpbox", "ssh"}, []int64{}, true}, - {"postgres", "e2-standard-2", []string{"postgres"}, []int64{}, true}, - {"ceph-1", "e2-standard-8", []string{"ceph"}, []int64{10, 100}, false}, - {"ceph-2", "e2-standard-8", []string{"ceph"}, []int64{10, 100}, false}, - {"ceph-3", "e2-standard-8", []string{"ceph"}, []int64{10, 100}, false}, - {"k0s-1", "e2-standard-8", []string{"k0s"}, []int64{}, false}, - {"k0s-2", "e2-standard-8", []string{"k0s"}, []int64{}, false}, - {"k0s-3", "e2-standard-8", []string{"k0s"}, []int64{}, false}, +// cephNodesPerDataCenter and k0sNodesPerDataCenter are the per-data-center node counts. Three +// Ceph nodes are the minimum for replication; three k0s nodes give one control plane and three +// workers, as written into the install config. +const ( + cephNodesPerDataCenter = 3 + k0sNodesPerDataCenter = 3 +) + +// sharedVMDefs returns the VMs that exist once per project, regardless of how many data centers +// are bootstrapped. The postgres node hosts the database both data centers share. +func sharedVMDefs() []VMDef { + return []VMDef{ + {Name: "jumpbox", MachineType: "e2-medium", Tags: []string{"jumpbox", "ssh"}, AdditionalDisks: []int64{}, ExternalIP: true}, + {Name: "postgres", MachineType: "e2-standard-2", Tags: []string{"postgres"}, AdditionalDisks: []int64{}, ExternalIP: true}, + } +} + +// dataCenterVMDefs returns the Ceph and k0s VMs of one data center. The suffix is empty for the +// primary data center, so single-DC bootstraps keep the names ceph-1..3 and k0s-1..3. +func dataCenterVMDefs(dcID int, suffix string) []VMDef { + defs := make([]VMDef, 0, cephNodesPerDataCenter+k0sNodesPerDataCenter) + for i := 1; i <= cephNodesPerDataCenter; i++ { + defs = append(defs, VMDef{ + Name: fmt.Sprintf("ceph-%d%s", i, suffix), + MachineType: "e2-standard-8", + Tags: []string{"ceph"}, + AdditionalDisks: []int64{10, 100}, + DataCenterID: dcID, + }) + } + for i := 1; i <= k0sNodesPerDataCenter; i++ { + defs = append(defs, VMDef{ + Name: fmt.Sprintf("k0s-%d%s", i, suffix), + MachineType: "e2-standard-8", + Tags: []string{"k0s"}, + AdditionalDisks: []int64{}, + DataCenterID: dcID, + }) + } + + return defs +} + +// VMDefsForEnv returns every VM definition of the environment: the project-shared VMs plus the +// Ceph and k0s VMs of each data center. When the environment carries no data centers — as with +// an infra file written before multi-DC support — it falls back to a single unsuffixed one. +func VMDefsForEnv(env *CodesphereEnvironment) []VMDef { + defs := sharedVMDefs() + + dcs := env.DataCenters + if len(dcs) == 0 { + dcs = []*datacenter.DataCenter{{ID: datacenter.PrimaryID}} + } + for _, dc := range dcs { + defs = append(defs, dataCenterVMDefs(dc.ID, dc.Suffix)...) + } + + return defs } // validateVMProvisioningOptions checks that spot and preemptible options are not both set @@ -51,16 +103,20 @@ type vmResult struct { name string externalIP string internalIP string + dcID int } // EnsureComputeInstances ensures that all required compute instances are present and running. func (b *GCPBootstrapper) EnsureComputeInstances() error { + b.ensureDataCenters() + + vms := VMDefsForEnv(b.Env) wg := sync.WaitGroup{} - errCh := make(chan error, len(vmDefs)) - resultCh := make(chan vmResult, len(vmDefs)) - logCh := make(chan string, len(vmDefs)) + errCh := make(chan error, len(vms)) + resultCh := make(chan vmResult, len(vms)) + logCh := make(chan string, len(vms)) - for _, vm := range vmDefs { + for _, vm := range vms { wg.Add(1) go func(vm VMDef) { defer wg.Done() @@ -95,6 +151,12 @@ func (b *GCPBootstrapper) EnsureComputeInstances() error { NodeClient: b.NodeClient, FileIO: b.fw, } + dcByID := map[int]*datacenter.DataCenter{} + for _, dc := range b.Env.DataCenters { + dc.CephNodes = nil + dc.ControlPlaneNodes = nil + dcByID[dc.ID] = dc + } for result := range resultCh { switch result.vmType { case "jumpbox": @@ -102,22 +164,31 @@ func (b *GCPBootstrapper) EnsureComputeInstances() error { case "postgres": b.Env.PostgreSQLNode = b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP) case "ceph": - node := b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP) - b.Env.CephNodes = append(b.Env.CephNodes, node) + dc, ok := dcByID[result.dcID] + if !ok { + return fmt.Errorf("instance %s belongs to unknown data center %d", result.name, result.dcID) + } + dc.CephNodes = append(dc.CephNodes, b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP)) case "k0s": - node := b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP) - b.Env.ControlPlaneNodes = append(b.Env.ControlPlaneNodes, node) + dc, ok := dcByID[result.dcID] + if !ok { + return fmt.Errorf("instance %s belongs to unknown data center %d", result.name, result.dcID) + } + dc.ControlPlaneNodes = append(dc.ControlPlaneNodes, b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP)) } } - //sort ceph nodes by name to ensure consistent ordering - sort.Slice(b.Env.CephNodes, func(i, j int) bool { - return b.Env.CephNodes[i].GetName() < b.Env.CephNodes[j].GetName() - }) - //sort control plane nodes by name to ensure consistent ordering - sort.Slice(b.Env.ControlPlaneNodes, func(i, j int) bool { - return b.Env.ControlPlaneNodes[i].GetName() < b.Env.ControlPlaneNodes[j].GetName() - }) + // Sort each data center's nodes by name to ensure consistent ordering, since the install + // config assigns roles by index. + for _, dc := range b.Env.DataCenters { + sort.Slice(dc.CephNodes, func(i, j int) bool { + return dc.CephNodes[i].GetName() < dc.CephNodes[j].GetName() + }) + sort.Slice(dc.ControlPlaneNodes, func(i, j int) bool { + return dc.ControlPlaneNodes[i].GetName() < dc.ControlPlaneNodes[j].GetName() + }) + } + b.mirrorPrimaryDataCenter() return nil } @@ -163,6 +234,7 @@ func (b *GCPBootstrapper) ensureVM(vm VMDef, rootDiskSize int64, logCh chan<- st name: vm.Name, externalIP: externalIP, internalIP: internalIP, + dcID: vm.DataCenterID, }, nil } @@ -362,30 +434,32 @@ func (b *GCPBootstrapper) waitForInstanceRunning(projectID, zone, name string, n name, pollInterval*time.Duration(maxAttempts)) } -// findVMDef looks up a VM definition by name. Returns nil if not found. -func findVMDef(name string) *VMDef { - for _, vm := range vmDefs { - if vm.Name == name { - return &vm +// findVMDef looks up a VM definition by name among the given definitions. Returns nil if not +// found. +func findVMDef(defs []VMDef, name string) *VMDef { + for i := range defs { + if defs[i].Name == name { + return &defs[i] } } return nil } -// validVMNames returns the list of known VM names from vmDefs. -func validVMNames() []string { - names := make([]string, len(vmDefs)) - for i, vm := range vmDefs { +// validVMNames returns the names of the given VM definitions. +func validVMNames(defs []VMDef) []string { + names := make([]string, len(defs)) + for i, vm := range defs { names[i] = vm.Name } return names } -// RestartVM restarts a single stopped or terminated VM by a name that is defined in vmDefs. +// RestartVM restarts a single stopped or terminated VM by a name defined for this environment. func (b *GCPBootstrapper) RestartVM(name string) error { - vm := findVMDef(name) + defs := VMDefsForEnv(b.Env) + vm := findVMDef(defs, name) if vm == nil { - return fmt.Errorf("unknown VM name %q; valid names are: %s", name, strings.Join(validVMNames(), ", ")) + return fmt.Errorf("unknown VM name %q; valid names are: %s", name, strings.Join(validVMNames(defs), ", ")) } projectID := b.Env.ProjectID @@ -424,10 +498,10 @@ func (b *GCPBootstrapper) RestartVM(name string) error { return nil } -// RestartVMs restarts all stopped or terminated VMs defined in vmDefs. +// RestartVMs restarts all stopped or terminated VMs of the environment, across every data center. func (b *GCPBootstrapper) RestartVMs() error { var errs []error - for _, vm := range vmDefs { + for _, vm := range VMDefsForEnv(b.Env) { if err := b.RestartVM(vm.Name); err != nil { errs = append(errs, err) } diff --git a/internal/bootstrap/gcp/gce_test.go b/internal/bootstrap/gcp/gce_test.go index 4d6a1f5f..73dee10c 100644 --- a/internal/bootstrap/gcp/gce_test.go +++ b/internal/bootstrap/gcp/gce_test.go @@ -23,6 +23,61 @@ import ( var _ = Describe("GCE", func() { + Describe("VMDefsForEnv", func() { + It("keeps the names a single-data-center bootstrap has always used", func() { + env := &gcp.CodesphereEnvironment{} + env.DataCenters = gcp.BuildDataCenters(env, nil) + + defs := gcp.VMDefsForEnv(env) + + Expect(vmNames(defs)).To(Equal([]string{ + "jumpbox", "postgres", + "ceph-1", "ceph-2", "ceph-3", + "k0s-1", "k0s-2", "k0s-3", + })) + }) + + It("adds suffixed ceph and k0s nodes per additional data center", func() { + env := &gcp.CodesphereEnvironment{MultiDC: true} + env.DataCenters = gcp.BuildDataCenters(env, nil) + + defs := gcp.VMDefsForEnv(env) + + Expect(vmNames(defs)).To(Equal([]string{ + "jumpbox", "postgres", + "ceph-1", "ceph-2", "ceph-3", + "k0s-1", "k0s-2", "k0s-3", + "ceph-1-dc2", "ceph-2-dc2", "ceph-3-dc2", + "k0s-1-dc2", "k0s-2-dc2", "k0s-3-dc2", + })) + }) + + It("assigns the shared VMs to no data center and the rest to theirs", func() { + env := &gcp.CodesphereEnvironment{MultiDC: true} + env.DataCenters = gcp.BuildDataCenters(env, nil) + + byName := map[string]int{} + for _, def := range gcp.VMDefsForEnv(env) { + byName[def.Name] = def.DataCenterID + } + + Expect(byName["jumpbox"]).To(BeZero()) + Expect(byName["postgres"]).To(BeZero()) + Expect(byName["ceph-1"]).To(Equal(1)) + Expect(byName["k0s-3"]).To(Equal(1)) + Expect(byName["ceph-1-dc2"]).To(Equal(2)) + Expect(byName["k0s-3-dc2"]).To(Equal(2)) + }) + + // Infra files written before multi-DC support carry no data center list. + It("falls back to a single unsuffixed data center when the environment has none", func() { + defs := gcp.VMDefsForEnv(&gcp.CodesphereEnvironment{}) + + Expect(vmNames(defs)).To(ContainElement("k0s-1")) + Expect(vmNames(defs)).To(HaveLen(8)) + }) + }) + Describe("IsNotFoundError", func() { Context("when error is nil", func() { It("should return false", func() { diff --git a/internal/bootstrap/gcp/test_helpers_test.go b/internal/bootstrap/gcp/test_helpers_test.go index e7131a4b..2d9629c6 100644 --- a/internal/bootstrap/gcp/test_helpers_test.go +++ b/internal/bootstrap/gcp/test_helpers_test.go @@ -24,6 +24,15 @@ import ( func protoString(s string) *string { return &s } +// vmNames returns the names of the given VM definitions, in order. +func vmNames(defs []gcp.VMDef) []string { + names := make([]string, len(defs)) + for i, def := range defs { + names[i] = def.Name + } + return names +} + // makeInstance creates a computepb.Instance with the given status and IPs. func makeInstance(status, internalIP, externalIP string) *computepb.Instance { inst := &computepb.Instance{