feat(gcp): derive the VM definitions per data center

cli/cmd/bootstrap_gcp_restart_vms.go

@@ -28,51 +28,61 @@ type BootstrapGcpRestartVMsOpts struct {
 	Name      string
 }
 
-// resolveProjectAndZone returns the project ID and zone from flags or the infra file.
-// If both flags are set they are used directly; if neither is set, the infra file is read.
-// Providing only one of --project-id / --zone is an error.
-func (c *BootstrapGcpRestartVMsCmd) resolveProjectAndZone(fw intutil.FileIO) (string, string, error) {
+// resolveEnvironment returns the environment to restart VMs in. Project ID and zone come from
+// the flags or, when neither is set, from the infra file. Providing only one of
+// --project-id / --zone is an error.
+//
+// The data center layout always comes from the infra file, since it determines the VM names. It
+// is read best-effort when the flags supply project and zone, in which case a missing file just
+// means single-data-center names.
+func (c *BootstrapGcpRestartVMsCmd) resolveEnvironment(fw intutil.FileIO) (*gcp.CodesphereEnvironment, error) {
 	projectID := c.Opts.ProjectID
 	zone := c.Opts.Zone
 
 	if (projectID == "") != (zone == "") {
-		return "", "", fmt.Errorf("--project-id and --zone must be provided together")
-	}
-	if projectID != "" {
-		return projectID, zone, nil
+		return nil, fmt.Errorf("--project-id and --zone must be provided together")
 	}
 
 	infraFilePath := gcp.GetInfraFilePath()
 	infraEnv, exists, err := gcp.LoadInfraFile(fw, infraFilePath)
 	if err != nil {
-		return "", "", fmt.Errorf("failed to load infra file: %w", err)
-	}
-	if !exists {
-		return "", "", fmt.Errorf("infra file not found at %s; use --project-id and --zone flags", infraFilePath)
+		if projectID == "" {
+			return nil, fmt.Errorf("failed to load infra file: %w", err)
+		}
+		log.Printf("Warning: %v", err)
 	}
-	if infraEnv.ProjectID == "" || infraEnv.Zone == "" {
-		return "", "", fmt.Errorf("infra file is missing project ID or zone; use --project-id and --zone flags")
+
+	if projectID == "" {
+		if !exists {
+			return nil, fmt.Errorf("infra file not found at %s; use --project-id and --zone flags", infraFilePath)
+		}
+		if infraEnv.ProjectID == "" || infraEnv.Zone == "" {
+			return nil, fmt.Errorf("infra file is missing project ID or zone; use --project-id and --zone flags")
+		}
+		projectID, zone = infraEnv.ProjectID, infraEnv.Zone
 	}
-	return infraEnv.ProjectID, infraEnv.Zone, nil
+
+	return &gcp.CodesphereEnvironment{
+		ProjectID:   projectID,
+		Zone:        zone,
+		MultiDC:     infraEnv.MultiDC,
+		DataCenters: infraEnv.DataCenters,
+	}, nil
 }
 
 func (c *BootstrapGcpRestartVMsCmd) RunE(_ *cobra.Command, _ []string) error {
 	ctx := c.cmd.Context()
 	stlog := bootstrap.NewStepLogger(false)
 	fw := intutil.NewFilesystemWriter()
 
-	projectID, zone, err := c.resolveProjectAndZone(fw)
+	csEnv, err := c.resolveEnvironment(fw)
 	if err != nil {
 		return err
 	}
+	projectID, zone := csEnv.ProjectID, csEnv.Zone
 
 	gcpClient := gcp.NewGCPClient(ctx, stlog, os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"))
 
-	csEnv := &gcp.CodesphereEnvironment{
-		ProjectID: projectID,
-		Zone:      zone,
-	}
-
 	bs, err := gcp.NewGCPBootstrapper(
 		ctx,
 		nil, stlog, csEnv, nil, gcpClient, fw, nil, nil, intutil.NewTime(), nil,
@@ -112,6 +122,7 @@ func AddBootstrapGcpRestartVMsCmd(bootstrapGcp *cobra.Command, opts *util.Global
 				{Desc: "Restart all VMs using project info from the local infra file"},
 				{Cmd: "--name jumpbox", Desc: "Restart only the jumpbox VM"},
 				{Cmd: "--name k0s-1", Desc: "Restart a specific k0s node"},
+				{Cmd: "--name k0s-1-dc2", Desc: "Restart a node of the second data center of a --multi-dc bootstrap"},
 				{Cmd: "--project-id my-project --zone us-central1-a", Desc: "Restart all VMs with explicit project and zone"},
 				{Cmd: "--project-id my-project --zone us-central1-a --name ceph-1", Desc: "Restart a specific VM with explicit project and zone"},
 			}),

docs/oms_beta_bootstrap-gcp_restart-vms.md

@@ -26,6 +26,9 @@ $ oms beta bootstrap-gcp restart-vms --name jumpbox
 # Restart a specific k0s node
 $ oms beta bootstrap-gcp restart-vms --name k0s-1
 
+# Restart a node of the second data center of a --multi-dc bootstrap
+$ oms beta bootstrap-gcp restart-vms --name k0s-1-dc2
+
 # Restart all VMs with explicit project and zone
 $ oms beta bootstrap-gcp restart-vms --project-id my-project --zone us-central1-a
 

internal/bootstrap/gcp/gce.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"cloud.google.com/go/compute/apiv1/computepb"
+	"github.com/codesphere-cloud/oms/internal/bootstrap/datacenter"
 	"github.com/codesphere-cloud/oms/internal/github"
 	"github.com/codesphere-cloud/oms/internal/installer/node"
 	"github.com/codesphere-cloud/oms/internal/util"
@@ -24,18 +25,69 @@ type VMDef struct {
 	Tags            []string
 	AdditionalDisks []int64
 	ExternalIP      bool
+	// DataCenterID is the data center the VM belongs to, or 0 for the project-shared VMs
+	// (jumpbox and postgres) that every data center uses.
+	DataCenterID int
 }
 
-// Example VM definitions (expand as needed)
-var vmDefs = []VMDef{
-	{"jumpbox", "e2-medium", []string{"jumpbox", "ssh"}, []int64{}, true},
-	{"postgres", "e2-standard-2", []string{"postgres"}, []int64{}, true},
-	{"ceph-1", "e2-standard-8", []string{"ceph"}, []int64{10, 100}, false},
-	{"ceph-2", "e2-standard-8", []string{"ceph"}, []int64{10, 100}, false},
-	{"ceph-3", "e2-standard-8", []string{"ceph"}, []int64{10, 100}, false},
-	{"k0s-1", "e2-standard-8", []string{"k0s"}, []int64{}, false},
-	{"k0s-2", "e2-standard-8", []string{"k0s"}, []int64{}, false},
-	{"k0s-3", "e2-standard-8", []string{"k0s"}, []int64{}, false},
+// cephNodesPerDataCenter and k0sNodesPerDataCenter are the per-data-center node counts. Three
+// Ceph nodes are the minimum for replication; three k0s nodes give one control plane and three
+// workers, as written into the install config.
+const (
+	cephNodesPerDataCenter = 3
+	k0sNodesPerDataCenter  = 3
+)
+
+// sharedVMDefs returns the VMs that exist once per project, regardless of how many data centers
+// are bootstrapped. The postgres node hosts the database both data centers share.
+func sharedVMDefs() []VMDef {
+	return []VMDef{
+		{Name: "jumpbox", MachineType: "e2-medium", Tags: []string{"jumpbox", "ssh"}, AdditionalDisks: []int64{}, ExternalIP: true},
+		{Name: "postgres", MachineType: "e2-standard-2", Tags: []string{"postgres"}, AdditionalDisks: []int64{}, ExternalIP: true},
+	}
+}
+
+// dataCenterVMDefs returns the Ceph and k0s VMs of one data center. The suffix is empty for the
+// primary data center, so single-DC bootstraps keep the names ceph-1..3 and k0s-1..3.
+func dataCenterVMDefs(dcID int, suffix string) []VMDef {
+	defs := make([]VMDef, 0, cephNodesPerDataCenter+k0sNodesPerDataCenter)
+	for i := 1; i <= cephNodesPerDataCenter; i++ {
+		defs = append(defs, VMDef{
+			Name:            fmt.Sprintf("ceph-%d%s", i, suffix),
+			MachineType:     "e2-standard-8",
+			Tags:            []string{"ceph"},
+			AdditionalDisks: []int64{10, 100},
+			DataCenterID:    dcID,
+		})
+	}
+	for i := 1; i <= k0sNodesPerDataCenter; i++ {
+		defs = append(defs, VMDef{
+			Name:            fmt.Sprintf("k0s-%d%s", i, suffix),
+			MachineType:     "e2-standard-8",
+			Tags:            []string{"k0s"},
+			AdditionalDisks: []int64{},
+			DataCenterID:    dcID,
+		})
+	}
+
+	return defs
+}
+
+// VMDefsForEnv returns every VM definition of the environment: the project-shared VMs plus the
+// Ceph and k0s VMs of each data center. When the environment carries no data centers — as with
+// an infra file written before multi-DC support — it falls back to a single unsuffixed one.
+func VMDefsForEnv(env *CodesphereEnvironment) []VMDef {
+	defs := sharedVMDefs()
+
+	dcs := env.DataCenters
+	if len(dcs) == 0 {
+		dcs = []*datacenter.DataCenter{{ID: datacenter.PrimaryID}}
+	}
+	for _, dc := range dcs {
+		defs = append(defs, dataCenterVMDefs(dc.ID, dc.Suffix)...)
+	}
+
+	return defs
 }
 
 // validateVMProvisioningOptions checks that spot and preemptible options are not both set
@@ -51,16 +103,20 @@ type vmResult struct {
 	name       string
 	externalIP string
 	internalIP string
+	dcID       int
 }
 
 // EnsureComputeInstances ensures that all required compute instances are present and running.
 func (b *GCPBootstrapper) EnsureComputeInstances() error {
+	b.ensureDataCenters()
+
+	vms := VMDefsForEnv(b.Env)
 	wg := sync.WaitGroup{}
-	errCh := make(chan error, len(vmDefs))
-	resultCh := make(chan vmResult, len(vmDefs))
-	logCh := make(chan string, len(vmDefs))
+	errCh := make(chan error, len(vms))
+	resultCh := make(chan vmResult, len(vms))
+	logCh := make(chan string, len(vms))
 
-	for _, vm := range vmDefs {
+	for _, vm := range vms {
 		wg.Add(1)
 		go func(vm VMDef) {
 			defer wg.Done()
@@ -95,29 +151,44 @@ func (b *GCPBootstrapper) EnsureComputeInstances() error {
 		NodeClient: b.NodeClient,
 		FileIO:     b.fw,
 	}
+	dcByID := map[int]*datacenter.DataCenter{}
+	for _, dc := range b.Env.DataCenters {
+		dc.CephNodes = nil
+		dc.ControlPlaneNodes = nil
+		dcByID[dc.ID] = dc
+	}
 	for result := range resultCh {
 		switch result.vmType {
 		case "jumpbox":
 			b.Env.Jumpbox.UpdateNode(result.name, result.externalIP, result.internalIP)
 		case "postgres":
 			b.Env.PostgreSQLNode = b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP)
 		case "ceph":
-			node := b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP)
-			b.Env.CephNodes = append(b.Env.CephNodes, node)
+			dc, ok := dcByID[result.dcID]
+			if !ok {
+				return fmt.Errorf("instance %s belongs to unknown data center %d", result.name, result.dcID)
+			}
+			dc.CephNodes = append(dc.CephNodes, b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP))
 		case "k0s":
-			node := b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP)
-			b.Env.ControlPlaneNodes = append(b.Env.ControlPlaneNodes, node)
+			dc, ok := dcByID[result.dcID]
+			if !ok {
+				return fmt.Errorf("instance %s belongs to unknown data center %d", result.name, result.dcID)
+			}
+			dc.ControlPlaneNodes = append(dc.ControlPlaneNodes, b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP))
 		}
 	}
 
-	//sort ceph nodes by name to ensure consistent ordering
-	sort.Slice(b.Env.CephNodes, func(i, j int) bool {
-		return b.Env.CephNodes[i].GetName() < b.Env.CephNodes[j].GetName()
-	})
-	//sort control plane nodes by name to ensure consistent ordering
-	sort.Slice(b.Env.ControlPlaneNodes, func(i, j int) bool {
-		return b.Env.ControlPlaneNodes[i].GetName() < b.Env.ControlPlaneNodes[j].GetName()
-	})
+	// Sort each data center's nodes by name to ensure consistent ordering, since the install
+	// config assigns roles by index.
+	for _, dc := range b.Env.DataCenters {
+		sort.Slice(dc.CephNodes, func(i, j int) bool {
+			return dc.CephNodes[i].GetName() < dc.CephNodes[j].GetName()
+		})
+		sort.Slice(dc.ControlPlaneNodes, func(i, j int) bool {
+			return dc.ControlPlaneNodes[i].GetName() < dc.ControlPlaneNodes[j].GetName()
+		})
+	}
+	b.mirrorPrimaryDataCenter()
 
 	return nil
 }
@@ -163,6 +234,7 @@ func (b *GCPBootstrapper) ensureVM(vm VMDef, rootDiskSize int64, logCh chan<- st
 		name:       vm.Name,
 		externalIP: externalIP,
 		internalIP: internalIP,
+		dcID:       vm.DataCenterID,
 	}, nil
 }
 
@@ -362,30 +434,32 @@ func (b *GCPBootstrapper) waitForInstanceRunning(projectID, zone, name string, n
 		name, pollInterval*time.Duration(maxAttempts))
 }
 
-// findVMDef looks up a VM definition by name. Returns nil if not found.
-func findVMDef(name string) *VMDef {
-	for _, vm := range vmDefs {
-		if vm.Name == name {
-			return &vm
+// findVMDef looks up a VM definition by name among the given definitions. Returns nil if not
+// found.
+func findVMDef(defs []VMDef, name string) *VMDef {
+	for i := range defs {
+		if defs[i].Name == name {
+			return &defs[i]
 		}
 	}
 	return nil
 }
 
-// validVMNames returns the list of known VM names from vmDefs.
-func validVMNames() []string {
-	names := make([]string, len(vmDefs))
-	for i, vm := range vmDefs {
+// validVMNames returns the names of the given VM definitions.
+func validVMNames(defs []VMDef) []string {
+	names := make([]string, len(defs))
+	for i, vm := range defs {
 		names[i] = vm.Name
 	}
 	return names
 }
 
-// RestartVM restarts a single stopped or terminated VM by a name that is defined in vmDefs.
+// RestartVM restarts a single stopped or terminated VM by a name defined for this environment.
 func (b *GCPBootstrapper) RestartVM(name string) error {
-	vm := findVMDef(name)
+	defs := VMDefsForEnv(b.Env)
+	vm := findVMDef(defs, name)
 	if vm == nil {
-		return fmt.Errorf("unknown VM name %q; valid names are: %s", name, strings.Join(validVMNames(), ", "))
+		return fmt.Errorf("unknown VM name %q; valid names are: %s", name, strings.Join(validVMNames(defs), ", "))
 	}
 
 	projectID := b.Env.ProjectID
@@ -424,10 +498,10 @@ func (b *GCPBootstrapper) RestartVM(name string) error {
 	return nil
 }
 
-// RestartVMs restarts all stopped or terminated VMs defined in vmDefs.
+// RestartVMs restarts all stopped or terminated VMs of the environment, across every data center.
 func (b *GCPBootstrapper) RestartVMs() error {
 	var errs []error
-	for _, vm := range vmDefs {
+	for _, vm := range VMDefsForEnv(b.Env) {
 		if err := b.RestartVM(vm.Name); err != nil {
 			errs = append(errs, err)
 		}

internal/bootstrap/gcp/gce_test.go

@@ -23,6 +23,61 @@ import (
 
 var _ = Describe("GCE", func() {
 
+	Describe("VMDefsForEnv", func() {
+		It("keeps the names a single-data-center bootstrap has always used", func() {
+			env := &gcp.CodesphereEnvironment{}
+			env.DataCenters = gcp.BuildDataCenters(env, nil)
+
+			defs := gcp.VMDefsForEnv(env)
+
+			Expect(vmNames(defs)).To(Equal([]string{
+				"jumpbox", "postgres",
+				"ceph-1", "ceph-2", "ceph-3",
+				"k0s-1", "k0s-2", "k0s-3",
+			}))
+		})
+
+		It("adds suffixed ceph and k0s nodes per additional data center", func() {
+			env := &gcp.CodesphereEnvironment{MultiDC: true}
+			env.DataCenters = gcp.BuildDataCenters(env, nil)
+
+			defs := gcp.VMDefsForEnv(env)
+
+			Expect(vmNames(defs)).To(Equal([]string{
+				"jumpbox", "postgres",
+				"ceph-1", "ceph-2", "ceph-3",
+				"k0s-1", "k0s-2", "k0s-3",
+				"ceph-1-dc2", "ceph-2-dc2", "ceph-3-dc2",
+				"k0s-1-dc2", "k0s-2-dc2", "k0s-3-dc2",
+			}))
+		})
+
+		It("assigns the shared VMs to no data center and the rest to theirs", func() {
+			env := &gcp.CodesphereEnvironment{MultiDC: true}
+			env.DataCenters = gcp.BuildDataCenters(env, nil)
+
+			byName := map[string]int{}
+			for _, def := range gcp.VMDefsForEnv(env) {
+				byName[def.Name] = def.DataCenterID
+			}
+
+			Expect(byName["jumpbox"]).To(BeZero())
+			Expect(byName["postgres"]).To(BeZero())
+			Expect(byName["ceph-1"]).To(Equal(1))
+			Expect(byName["k0s-3"]).To(Equal(1))
+			Expect(byName["ceph-1-dc2"]).To(Equal(2))
+			Expect(byName["k0s-3-dc2"]).To(Equal(2))
+		})
+
+		// Infra files written before multi-DC support carry no data center list.
+		It("falls back to a single unsuffixed data center when the environment has none", func() {
+			defs := gcp.VMDefsForEnv(&gcp.CodesphereEnvironment{})
+
+			Expect(vmNames(defs)).To(ContainElement("k0s-1"))
+			Expect(vmNames(defs)).To(HaveLen(8))
+		})
+	})
+
 	Describe("IsNotFoundError", func() {
 		Context("when error is nil", func() {
 			It("should return false", func() {