diff --git a/internal/bootstrap/gcp/multi_dc_test.go b/internal/bootstrap/gcp/multi_dc_test.go new file mode 100644 index 00000000..3f62f3fc --- /dev/null +++ b/internal/bootstrap/gcp/multi_dc_test.go @@ -0,0 +1,327 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package gcp_test + +import ( + "context" + "fmt" + "os" + + "cloud.google.com/go/compute/apiv1/computepb" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" + "google.golang.org/api/cloudbilling/v1" + + "github.com/codesphere-cloud/oms/internal/bootstrap" + "github.com/codesphere-cloud/oms/internal/bootstrap/gcp" + "github.com/codesphere-cloud/oms/internal/codesphere" + "github.com/codesphere-cloud/oms/internal/env" + "github.com/codesphere-cloud/oms/internal/github" + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" + "github.com/codesphere-cloud/oms/internal/installer/node" + "github.com/codesphere-cloud/oms/internal/portal" + "github.com/codesphere-cloud/oms/internal/util" +) + +// realConfigManager returns an install config manager that keeps its files in memory, so tests +// can inspect the config and vault a data center would actually be installed with. +func realConfigManager(writes map[string][]byte) installer.InstallConfigManager { + icm := installer.NewInstallConfigManager() + icm.(*installer.InstallConfig).SetFileIO(&recordingFileIO{writes: writes}) + return icm +} + +// recordingFileIO records written files and reports every other path as missing, so a bootstrap +// run behaves as if nothing existed locally beforehand. +type recordingFileIO struct { + util.FilesystemWriter + writes map[string][]byte +} + +func (f *recordingFileIO) CreateAndWrite(path string, content []byte, _ string) error { + f.writes[path] = content + return nil +} + +var _ = Describe("Multi-DC bootstrap", func() { + var ( + nodeClient *node.MockNodeClient + csEnv *gcp.CodesphereEnvironment + gc *gcp.MockGCPClientManager + fw *util.MockFileIO + writes map[string][]byte + primaryICG installer.InstallConfigManager + + bs *gcp.GCPBootstrapper + ) + + BeforeEach(func() { + nodeClient = node.NewMockNodeClient(GinkgoT()) + gc = gcp.NewMockGCPClientManager(GinkgoT()) + fw = util.NewMockFileIO(GinkgoT()) + writes = map[string][]byte{} + primaryICG = realConfigManager(writes) + + csEnv = &gcp.CodesphereEnvironment{ + MultiDC: true, + ProjectName: "test-project", + BillingAccount: "test-billing-account", + BaseDomain: "example.com", + Region: "us-central1", + Zone: "us-central1-a", + DNSProjectID: "dns-project", + DNSZoneName: "test-zone", + ProjectTTL: "1h", + SecretsDir: "/etc/codesphere/secrets", + DatacenterName: "multidc", + InstallConfigPath: "config.yaml", + SecretsFilePath: "prod.vault.yaml", + WriteConfig: true, + RegistryType: gcp.RegistryTypeGitHub, + RegistryUser: "registry-user", + GitHubPAT: "fake-pat", + SSHPublicKeyPath: "key.pub", + RootDiskSize: 50, + InternalFlags: gcp.DefaultInternalFlags, + PreviewFlags: gcp.DefaultPreviewFlags, + FeatureFlags: gcp.DefaultFeatureFlags, + } + }) + + JustBeforeEach(func() { + var err error + bs, err = gcp.NewGCPBootstrapper( + context.Background(), + env.NewEnv(), + bootstrap.NewStepLogger(false), + csEnv, + primaryICG, + gc, + fw, + nodeClient, + portal.NewMockPortal(GinkgoT()), + util.NewFakeTime(), + github.NewMockGitHubClient(GinkgoT()), + ) + Expect(err).NotTo(HaveOccurred()) + bs.NewConfigManager = func() installer.InstallConfigManager { return realConfigManager(writes) } + }) + + // expectBootstrapMocks sets up the GCP, SSH and file mocks for a full two-data-center run. + expectBootstrapMocks := func(projectID string) { + const vmCount = 14 + + fw.EXPECT().Exists(mock.Anything).Return(false) + fw.EXPECT().MkdirAll(mock.Anything, os.FileMode(0755)).Return(nil) + fw.EXPECT().WriteFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + fw.EXPECT().ReadFile(mock.Anything).Return([]byte("ssh-rsa AAA..."), nil).Times(vmCount) + + // EnsureProject only creates a project when the lookup fails with this exact message. + gc.EXPECT().GetProjectByName(mock.Anything, "test-project").Return(nil, fmt.Errorf("project not found: test-project")) + gc.EXPECT().CreateProjectID("test-project").Return(projectID) + gc.EXPECT().CreateProject(mock.Anything, mock.Anything, "test-project", mock.Anything).Return(mock.Anything, nil) + gc.EXPECT().GetBillingInfo(projectID).Return(&cloudbilling.ProjectBillingInfo{BillingEnabled: false}, nil) + gc.EXPECT().EnableBilling(projectID, "test-billing-account").Return(nil) + gc.EXPECT().EnableAPIs(projectID, mock.Anything).Return(nil) + gc.EXPECT().CreateServiceAccount(projectID, "cloud-controller", "cloud-controller").Return("cc@p.iam.gserviceaccount.com", false, nil) + gc.EXPECT().AssignIAMRole(projectID, "cloud-controller", projectID, []string{"roles/compute.admin"}).Return(nil) + gc.EXPECT().AssignIAMRole("dns-project", "cloud-controller", projectID, []string{"roles/dns.admin"}).Return(nil) + gc.EXPECT().CreateVPC(projectID, "us-central1", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + gc.EXPECT().CreateFirewallRule(projectID, mock.Anything).Return(nil).Times(5) + + mockGetInstanceNotFoundThenRunning(gc, projectID, "us-central1-a", makeRunningInstance("10.10.0.2", "1.2.3.4"), vmCount) + gc.EXPECT().CreateInstance(projectID, "us-central1-a", mock.Anything).Return(nil).Times(vmCount) + + // Two data centers reserve three addresses each, suffixed for the second one. + for _, name := range []string{ + "gateway", "public-gateway", "ssh-proxy", + "gateway-dc2", "public-gateway-dc2", "ssh-proxy-dc2", + } { + ip := fmt.Sprintf("203.0.113.%d", len(name)) + gc.EXPECT().GetAddress(projectID, "us-central1", name).Return(nil, fmt.Errorf("not found")).Once() + gc.EXPECT().CreateAddress(projectID, "us-central1", mock.MatchedBy(func(addr *computepb.Address) bool { + return addr.GetName() == name + })).Return(ip, nil).Once() + } + + gc.EXPECT().EnsureDNSManagedZone("dns-project", "test-zone", "example.com.", mock.Anything).Return(nil) + gc.EXPECT().EnsureDNSRecordSets("dns-project", "test-zone", mock.Anything).Return(nil) + + nodeClient.EXPECT().WaitReady(mock.Anything, mock.Anything).Return(nil) + nodeClient.EXPECT().HasFile(mock.Anything, mock.Anything).Return(false) + nodeClient.EXPECT().RunCommand(mock.Anything, mock.Anything, mock.Anything).Return(nil) + nodeClient.EXPECT().CopyFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + } + + It("bootstraps two data centers sharing one database", func() { + expectBootstrapMocks("test-project-12345") + + Expect(bs.Bootstrap()).To(Succeed()) + + Expect(bs.Env.DataCenters).To(HaveLen(2)) + primary, secondary := bs.Env.DataCenters[0], bs.Env.DataCenters[1] + + By("giving each data center its own ceph and k0s nodes") + Expect(vmNamesOfNodes(primary.CephNodes)).To(Equal([]string{"ceph-1", "ceph-2", "ceph-3"})) + Expect(vmNamesOfNodes(primary.ControlPlaneNodes)).To(Equal([]string{"k0s-1", "k0s-2", "k0s-3"})) + Expect(vmNamesOfNodes(secondary.CephNodes)).To(Equal([]string{"ceph-1-dc2", "ceph-2-dc2", "ceph-3-dc2"})) + Expect(vmNamesOfNodes(secondary.ControlPlaneNodes)).To(Equal([]string{"k0s-1-dc2", "k0s-2-dc2", "k0s-3-dc2"})) + + By("sharing the platform domain and scoping the workspace domain") + Expect(primary.InstallConfig.Codesphere.Domain).To(Equal("cs.example.com")) + Expect(secondary.InstallConfig.Codesphere.Domain).To(Equal("cs.example.com")) + Expect(primary.InstallConfig.Codesphere.WorkspaceHostingBaseDomain).To(Equal("1.ws.example.com")) + Expect(secondary.InstallConfig.Codesphere.WorkspaceHostingBaseDomain).To(Equal("2.ws.example.com")) + Expect(secondary.InstallConfig.Codesphere.CustomDomains.CNameBaseDomain).To(Equal("2.ws.example.com")) + + By("giving each data center a distinct ID and name") + Expect(primary.InstallConfig.Datacenter.ID).To(Equal(1)) + Expect(secondary.InstallConfig.Datacenter.ID).To(Equal(2)) + // The k0s cluster is named codesphere-, so the names must differ. + Expect(primary.InstallConfig.Datacenter.Name).To(Equal("multidc")) + Expect(secondary.InstallConfig.Datacenter.Name).To(Equal("multidc-dc2")) + + By("installing postgres in the primary data center and reusing it in the secondary") + Expect(primary.InstallConfig.Postgres.Mode).To(Equal("install")) + Expect(primary.InstallConfig.Postgres.Primary.IP).To(Equal("10.10.0.2")) + Expect(secondary.InstallConfig.Postgres.Mode).To(Equal("external")) + // The server certificate carries only an IP SAN, so the address must be the IP. + Expect(secondary.InstallConfig.Postgres.ServerAddress).To(Equal("10.10.0.2")) + Expect(secondary.InstallConfig.Postgres.Primary).To(BeNil()) + Expect(secondary.InstallConfig.Postgres.Replica).To(BeNil()) + Expect(secondary.InstallConfig.Postgres.CACertPem).To(Equal(primary.InstallConfig.Postgres.CACertPem)) + Expect(secondary.InstallConfig.Postgres.CACertPem).NotTo(BeEmpty()) + + By("skipping the postgres install step in the secondary data center only") + Expect(secondary.InstallConfig.Operations.Skip).To(ContainElement("postgres")) + // Both data centers install their own ceph and kubernetes. + Expect(secondary.InstallConfig.Operations.Skip).NotTo(ContainElement("ceph")) + Expect(secondary.InstallConfig.Operations.Skip).NotTo(ContainElement("kubernetes")) + if primary.InstallConfig.Operations != nil { + Expect(primary.InstallConfig.Operations.Skip).NotTo(ContainElement("postgres")) + } + + By("telling both data centers about every data center of the installation") + // The installer defaults dataCenters to the local one, so without the list each data + // center renders as a single-data-center instance. + topology := []files.DatacenterConfig{ + {ID: 1, Name: "multidc", City: "Karlsruhe", CountryCode: "DE"}, + {ID: 2, Name: "multidc-dc2", City: "Karlsruhe", CountryCode: "DE"}, + } + for _, dc := range bs.Env.DataCenters { + Expect(dc.InstallConfig.DataCenters).To(Equal(topology), "data center %d", dc.ID) + Expect(dc.InstallConfig.DefaultDataCenterID).To(Equal(1), "data center %d", dc.ID) + // The local data center is still the one this config installs. + Expect(dc.InstallConfig.Datacenter.ID).To(Equal(dc.ID)) + } + + By("giving each data center its own vault directory on the shared jumpbox") + Expect(primary.InstallConfig.Secrets.BaseDir).To(Equal("/etc/codesphere/secrets")) + Expect(secondary.InstallConfig.Secrets.BaseDir).To(Equal("/etc/codesphere/secrets-dc2")) + Expect(writes).To(HaveKey("config.yaml")) + Expect(writes).To(HaveKey("config-dc2.yaml")) + Expect(writes).To(HaveKey("prod.vault.yaml")) + Expect(writes).To(HaveKey("prod-dc2.vault.yaml")) + + primaryVault := primary.ConfigManager().GetVault() + secondaryVault := secondary.ConfigManager().GetVault() + + By("sharing every secret both data centers need to agree on") + shared := []string{ + files.SecretPostgresPassword, + files.SecretPostgresReplicaPassword, + files.SecretPostgresCaKeyPem, + files.SecretTokenPrivateKey, + files.SecretTokenPublicKey, + files.SecretDomainAuthPrivateKey, + files.SecretMounterHmacSecret, + files.SecretMongoDbPasswordEncryptionKey, + } + for _, svc := range codesphere.PostgresServices { + shared = append(shared, files.PostgresUserSecretName(svc.Name), files.PostgresPasswordSecretName(svc.Name)) + } + for _, name := range shared { + Expect(primaryVault.GetSecret(name)).NotTo(BeNil(), "primary should have %s", name) + Expect(secondaryVault.GetSecret(name)).To(Equal(primaryVault.GetSecret(name)), "%s must be shared", name) + } + + By("regenerating the per-cluster secrets for the secondary data center") + for _, name := range []string{files.SecretSelfSignedCaKeyPem, files.SecretCephSshPrivateKey} { + Expect(secondaryVault.GetSecret(name)).NotTo(BeNil(), "secondary should have %s", name) + Expect(secondaryVault.GetSecret(name).File.Content). + NotTo(Equal(primaryVault.GetSecret(name).File.Content), "%s must be per data center", name) + } + Expect(secondary.InstallConfig.Cluster.Certificates.CA.CertPem). + NotTo(Equal(primary.InstallConfig.Cluster.Certificates.CA.CertPem)) + Expect(secondary.InstallConfig.Ceph.CephAdmSSHKey.PublicKey). + NotTo(Equal(primary.InstallConfig.Ceph.CephAdmSSHKey.PublicKey)) + + By("recording the DNS records it created so cleanup can delete them") + Expect(bs.Env.DNSRecords).To(Equal(gcp.DataCenterDNSRecordNames("example.com", bs.Env.DataCenters))) + + By("pointing each data center's install command at its own config and vault") + Expect(bs.InstallCommand(primary, "pkg.tar.gz")).To(ContainSubstring("-c /etc/codesphere/config.yaml")) + Expect(bs.InstallCommand(primary, "pkg.tar.gz")).To(ContainSubstring("--vault /etc/codesphere/secrets/prod.vault.yaml")) + Expect(bs.InstallCommand(secondary, "pkg.tar.gz")).To(ContainSubstring("-c /etc/codesphere/config-dc2.yaml")) + Expect(bs.InstallCommand(secondary, "pkg.tar.gz")).To(ContainSubstring("--vault /etc/codesphere/secrets-dc2/prod.vault.yaml")) + + By("producing postgres blocks the installer's validation accepts") + for _, dc := range bs.Env.DataCenters { + for _, problem := range dc.ConfigManager().ValidateInstallConfig() { + Expect(problem).NotTo(ContainSubstring("postgres"), "data center %d", dc.ID) + } + } + }) + + It("mirrors the primary data center onto the legacy infra file fields", func() { + expectBootstrapMocks("test-project-12345") + + Expect(bs.Bootstrap()).To(Succeed()) + + primary := bs.Env.DataCenters[0] + Expect(bs.Env.ControlPlaneNodes).To(Equal(primary.ControlPlaneNodes)) + Expect(bs.Env.CephNodes).To(Equal(primary.CephNodes)) + Expect(bs.Env.GatewayIP).To(Equal(primary.GatewayIP)) + Expect(bs.Env.PublicGatewayIP).To(Equal(primary.PublicGatewayIP)) + Expect(bs.Env.SshProxyIP).To(Equal(primary.SshProxyIP)) + }) + + Describe("validateMultiDC", func() { + DescribeTable("rejects flag combinations it cannot satisfy", + func(mutate func(), wantErr string) { + mutate() + Expect(bs.ValidateInput()).To(MatchError(ContainSubstring(wantErr))) + }, + Entry("without write-config", func() { csEnv.WriteConfig = false }, "multi-dc requires write-config"), + Entry("with an explicit datacenter ID", func() { csEnv.DatacenterIDExplicit = true }, "datacenter-id cannot be combined with multi-dc"), + Entry("without a datacenter name", func() { csEnv.DatacenterName = "" }, "datacenter-name is required with multi-dc"), + Entry("without a config path", func() { csEnv.InstallConfigPath = "" }, "cannot derive a per-data-center path"), + Entry("without a secrets path", func() { csEnv.SecretsFilePath = "" }, "cannot derive a per-data-center path"), + ) + + It("accepts the default multi-dc flags", func() { + Expect(bs.ValidateInput()).To(Succeed()) + }) + + It("does not constrain a single-data-center bootstrap", func() { + csEnv.MultiDC = false + csEnv.WriteConfig = false + csEnv.DatacenterName = "" + csEnv.DatacenterIDExplicit = true + + Expect(bs.ValidateInput()).To(Succeed()) + }) + }) +}) + +// vmNamesOfNodes returns the names of the given nodes, in order. +func vmNamesOfNodes(nodes []*node.Node) []string { + names := make([]string, len(nodes)) + for i, n := range nodes { + names[i] = n.GetName() + } + return names +}