feat(installer): derive a data center's vault from the primary's

internal/installer/secrets/datacenter.go

@@ -0,0 +1,107 @@
+// Copyright (c) Codesphere Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package secrets
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/codesphere-cloud/oms/internal/installer/files"
+)
+
+// DataCenterScopedSecretNames are the secrets that belong to exactly one data center and must
+// therefore be regenerated for every additional data center of a multi-DC installation.
+//
+// Everything not listed here (and not matched by DataCenterScopedSecretPrefixes) is shared:
+// the postgres roles because both data centers talk to the same server, and the auth and
+// encryption keys because tokens minted and rows written in one data center are consumed in
+// the other.
+var DataCenterScopedSecretNames = []string{
+	// Cluster ingress CA — paired with cluster.certificates.ca.certPem.
+	files.SecretSelfSignedCaKeyPem,
+	// cephadm SSH key — paired with ceph.cephAdmSshKey.publicKey.
+	files.SecretCephSshPrivateKey,
+	// Written by the installer's kubernetes step; points at one cluster's API server.
+	files.SecretKubeConfig,
+	// ACME external account binding — paired with codesphere.certIssuer.acme.eabKeyId.
+	files.SecretAcmeEabMacKey,
+	// Only present in recovered vaults; keyed to a single cluster's nix cache.
+	files.SecretPrivNixSigningKey,
+	files.SecretPubNixSigningKey,
+}
+
+// DataCenterScopedSecretPrefixes cover the Ceph cluster credentials that the installer's ceph
+// step writes back into the vault (cephFsId, cephfsAdmin, csiRbdNode, rgwAdminAccessKey, ...).
+// The installer owns those names, so they are matched by prefix rather than enumerated.
+var DataCenterScopedSecretPrefixes = []string{"ceph", "csi", "rgw"}
+
+// IsDataCenterScopedSecret reports whether a vault entry belongs to a single data center.
+func IsDataCenterScopedSecret(name string) bool {
+	for _, scoped := range DataCenterScopedSecretNames {
+		if name == scoped {
+			return true
+		}
+	}
+	for _, prefix := range DataCenterScopedSecretPrefixes {
+		if strings.HasPrefix(name, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
+// DeriveDataCenterVault returns a copy of the primary data center's vault with all
+// data-center-scoped secrets removed. Running EnsureSecrets over the result regenerates exactly
+// those, while the shared secrets stay byte-identical across data centers.
+//
+// Derive the config with DeriveDataCenterConfig before running EnsureSecrets, so the config
+// fields paired with the dropped secrets are regenerated along with them.
+func DeriveDataCenterVault(primary *files.InstallVault) *files.InstallVault {
+	derived := primary.Clone()
+	if derived == nil {
+		return nil
+	}
+
+	kept := make([]files.SecretEntry, 0, len(derived.Secrets))
+	for _, entry := range derived.Secrets {
+		if IsDataCenterScopedSecret(entry.Name) {
+			continue
+		}
+		kept = append(kept, entry)
+	}
+	derived.Secrets = kept
+
+	return derived
+}
+
+// DeriveDataCenterConfig returns a copy of the primary data center's config with the fields that
+// belong to a single data center reset, so EnsureSecrets fills them in for this data center.
+//
+// The counterpart of DeriveDataCenterVault: both derive from the primary data center and both run
+// before EnsureSecrets, which regenerates every dropped secret together with the config field
+// paired with it.
+func DeriveDataCenterConfig(primary *files.RootConfig) (*files.RootConfig, error) {
+	derived, err := primary.Clone()
+	if err != nil {
+		return nil, fmt.Errorf("clone primary data center config: %w", err)
+	}
+	clearDataCenterScopedConfig(derived)
+
+	return derived, nil
+}
+
+// clearDataCenterScopedConfig resets the config fields that are written by the Ensure* functions
+// alongside a data-center-scoped vault secret. Those pairs must always be mutated together: the
+// Ensure* functions are gated on the vault entry, so a config field left in place would keep a
+// stale value paired with a freshly generated key.
+func clearDataCenterScopedConfig(config *files.RootConfig) {
+	// Paired with selfSignedCaKeyPem, written by EnsureIngressCA.
+	config.Cluster.Certificates.CA.CertPem = ""
+	// Paired with cephSshPrivateKey, written by EnsureCephSSHKeys.
+	config.Ceph.CephAdmSSHKey.PublicKey = ""
+	// Paired with acmeEabMacKey, obtained per data center from the ACME CA.
+	if config.Codesphere.CertIssuer.Acme != nil {
+		config.Codesphere.CertIssuer.Acme.EABKeyID = ""
+	}
+}

internal/installer/secrets/datacenter_test.go

@@ -0,0 +1,239 @@
+// Copyright (c) Codesphere Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package secrets_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/codesphere-cloud/oms/internal/installer/files"
+	"github.com/codesphere-cloud/oms/internal/installer/secrets"
+)
+
+// primaryConfig returns a config shaped like the one the GCP bootstrapper generates for the
+// primary data center: postgres installed on a dedicated node, so EnsureSecrets produces the
+// full postgres secret set.
+func primaryConfig() *files.RootConfig {
+	return &files.RootConfig{
+		Postgres: files.PostgresConfig{
+			Mode: "install",
+			Primary: &files.PostgresPrimaryConfig{
+				IP:       "10.10.0.5",
+				Hostname: "postgres",
+			},
+		},
+		Codesphere: files.CodesphereConfig{
+			CertIssuer: files.CertIssuerConfig{
+				Type: "acme",
+				Acme: &files.ACMEConfig{Enabled: true, EABKeyID: "primary-eab-key"},
+			},
+		},
+	}
+}
+
+var _ = Describe("IsDataCenterScopedSecret", func() {
+	DescribeTable("classifies vault secrets",
+		func(name string, expected bool) {
+			Expect(secrets.IsDataCenterScopedSecret(name)).To(Equal(expected))
+		},
+		Entry("cluster ingress CA key", files.SecretSelfSignedCaKeyPem, true),
+		Entry("cephadm ssh key", files.SecretCephSshPrivateKey, true),
+		Entry("kubeconfig", files.SecretKubeConfig, true),
+		Entry("acme EAB mac key", files.SecretAcmeEabMacKey, true),
+		Entry("nix signing key", files.SecretPrivNixSigningKey, true),
+		Entry("ceph fs id", "cephFsId", true),
+		Entry("cephfs admin", "cephfsAdminCodesphere", true),
+		Entry("csi provisioner", "csiRbdProvisioner", true),
+		Entry("rgw admin key", "rgwAdminAccessKey", true),
+
+		Entry("postgres admin password", files.SecretPostgresPassword, false),
+		Entry("postgres CA key", files.SecretPostgresCaKeyPem, false),
+		Entry("postgres primary server key", files.SecretPostgresPrimaryServerKeyPem, false),
+		Entry("per-service postgres user", "postgresUserAuth", false),
+		Entry("per-service postgres password", "postgresPasswordAuth", false),
+		Entry("token signing key", files.SecretTokenPrivateKey, false),
+		Entry("domain auth key", files.SecretDomainAuthPrivateKey, false),
+		Entry("mounter hmac", files.SecretMounterHmacSecret, false),
+		Entry("managed service password encryption key", files.SecretMongoDbPasswordEncryptionKey, false),
+		Entry("registry password", files.SecretRegistryPassword, false),
+		Entry("ssh workspace proxy host key", files.SecretSshWorkspaceProxyHostKey, false),
+	)
+})
+
+var _ = Describe("DeriveDataCenterVault", func() {
+	var primary *files.InstallVault
+
+	BeforeEach(func() {
+		primary = newVault()
+		Expect(secrets.EnsureSecrets(primary, primaryConfig())).To(Succeed())
+		// Simulate the ceph and kubernetes install steps writing their credentials back.
+		primary.SetSecret(files.SecretEntry{Name: "cephFsId", Fields: &files.SecretFields{Password: "fsid-1"}})
+		primary.SetSecret(files.SecretEntry{Name: "csiRbdNode", Fields: &files.SecretFields{Password: "csi-1"}})
+		primary.SetSecret(files.SecretEntry{Name: "rgwAdminSecretKey", Fields: &files.SecretFields{Password: "rgw-1"}})
+		primary.SetSecret(files.SecretEntry{Name: files.SecretKubeConfig, File: &files.SecretFile{Name: "kubeConfig", Content: "dc1-kubeconfig"}})
+	})
+
+	It("keeps the secrets that both data centers must share", func() {
+		derived := secrets.DeriveDataCenterVault(primary)
+
+		shared := []string{
+			files.SecretPostgresPassword,
+			files.SecretPostgresReplicaPassword,
+			files.SecretPostgresCaKeyPem,
+			files.SecretPostgresPrimaryServerKeyPem,
+			files.SecretPostgresReplicaServerKeyPem,
+			files.SecretTokenPrivateKey,
+			files.SecretTokenPublicKey,
+			files.SecretDomainAuthPrivateKey,
+			files.SecretDomainAuthPublicKey,
+			files.SecretMounterHmacSecret,
+			files.SecretMongoDbPasswordEncryptionKey,
+			files.SecretSshWorkspaceProxyHostKey,
+		}
+		for _, name := range shared {
+			original := primary.GetSecret(name)
+			Expect(original).ToNot(BeNil(), "primary vault should contain %s", name)
+			Expect(derived.GetSecret(name)).To(Equal(original), "%s should be shared", name)
+		}
+	})
+
+	It("keeps every per-service postgres role, because the roles live on the shared server", func() {
+		derived := secrets.DeriveDataCenterVault(primary)
+
+		for _, svc := range []string{"Auth", "Deployment", "Ide", "Marketplace", "Payment", "PublicApi", "Team", "Workspace"} {
+			for _, prefix := range []string{"postgresUser", "postgresPassword"} {
+				name := prefix + svc
+				if primary.GetSecret(name) == nil {
+					continue
+				}
+				Expect(derived.GetSecret(name)).To(Equal(primary.GetSecret(name)), "%s should be shared", name)
+			}
+		}
+	})
+
+	It("drops the data-center-scoped secrets", func() {
+		derived := secrets.DeriveDataCenterVault(primary)
+
+		for _, name := range []string{
+			files.SecretSelfSignedCaKeyPem,
+			files.SecretCephSshPrivateKey,
+			files.SecretKubeConfig,
+			"cephFsId",
+			"csiRbdNode",
+			"rgwAdminSecretKey",
+		} {
+			Expect(derived.GetSecret(name)).To(BeNil(), "%s should be dropped", name)
+		}
+	})
+
+	It("does not alias the primary vault", func() {
+		derived := secrets.DeriveDataCenterVault(primary)
+
+		derived.GetSecret(files.SecretPostgresPassword).Fields.Password = "mutated"
+		Expect(primary.GetSecret(files.SecretPostgresPassword).Fields.Password).ToNot(Equal("mutated"))
+	})
+
+	It("returns nil for a nil vault", func() {
+		Expect(secrets.DeriveDataCenterVault(nil)).To(BeNil())
+	})
+})
+
+var _ = Describe("DeriveDataCenterConfig", func() {
+	var primary *files.RootConfig
+
+	BeforeEach(func() {
+		primary = primaryConfig()
+		primary.Cluster.Certificates.CA.CertPem = "primary-ca-cert"
+		primary.Ceph.CephAdmSSHKey.PublicKey = "ssh-rsa primary-cephadm"
+		primary.Postgres.CACertPem = "primary-postgres-ca"
+		primary.Codesphere.Domain = "cs.example.com"
+	})
+
+	It("resets the fields paired with a data-center-scoped secret", func() {
+		derived, err := secrets.DeriveDataCenterConfig(primary)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(derived.Cluster.Certificates.CA.CertPem).To(BeEmpty())
+		Expect(derived.Ceph.CephAdmSSHKey.PublicKey).To(BeEmpty())
+		Expect(derived.Codesphere.CertIssuer.Acme.EABKeyID).To(BeEmpty())
+	})
+
+	It("keeps the fields shared by every data center", func() {
+		derived, err := secrets.DeriveDataCenterConfig(primary)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(derived.Postgres.CACertPem).To(Equal("primary-postgres-ca"))
+		Expect(derived.Codesphere.Domain).To(Equal("cs.example.com"))
+		Expect(derived.Codesphere.CertIssuer.Type).To(Equal(files.CertIssuerTypeACME))
+		Expect(derived.Codesphere.CertIssuer.Acme.Enabled).To(BeTrue())
+	})
+
+	It("does not modify or alias the primary config", func() {
+		derived, err := secrets.DeriveDataCenterConfig(primary)
+		Expect(err).NotTo(HaveOccurred())
+
+		derived.Codesphere.CertIssuer.Acme.Server = "https://other.example.com/directory"
+
+		Expect(primary.Cluster.Certificates.CA.CertPem).To(Equal("primary-ca-cert"))
+		Expect(primary.Ceph.CephAdmSSHKey.PublicKey).To(Equal("ssh-rsa primary-cephadm"))
+		Expect(primary.Codesphere.CertIssuer.Acme.EABKeyID).To(Equal("primary-eab-key"))
+		Expect(primary.Codesphere.CertIssuer.Acme.Server).ToNot(Equal("https://other.example.com/directory"))
+	})
+
+	It("handles a config without an ACME issuer", func() {
+		primary.Codesphere.CertIssuer = files.CertIssuerConfig{Type: files.CertIssuerTypeSelfSigned}
+
+		derived, err := secrets.DeriveDataCenterConfig(primary)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(derived.Codesphere.CertIssuer.Acme).To(BeNil())
+		Expect(derived.Cluster.Certificates.CA.CertPem).To(BeEmpty())
+	})
+})
+
+var _ = Describe("secondary data center secret generation", func() {
+	It("shares the database and auth secrets and regenerates the per-cluster ones", func() {
+		primaryVault := newVault()
+		primaryCfg := primaryConfig()
+		Expect(secrets.EnsureSecrets(primaryVault, primaryCfg)).To(Succeed())
+
+		secondaryVault := secrets.DeriveDataCenterVault(primaryVault)
+		secondaryCfg, err := secrets.DeriveDataCenterConfig(primaryCfg)
+		Expect(err).NotTo(HaveOccurred())
+		// The bootstrapper points a secondary data center at the shared postgres server before
+		// generating its secrets, so no postgres secret is regenerated here.
+		secondaryCfg.Postgres = files.PostgresConfig{
+			Mode:          "external",
+			ServerAddress: "10.10.0.5",
+			CACertPem:     primaryCfg.Postgres.CACertPem,
+		}
+		Expect(secrets.EnsureSecrets(secondaryVault, secondaryCfg)).To(Succeed())
+
+		By("keeping the shared postgres and auth credentials byte-identical")
+		Expect(secondaryVault.GetSecret(files.SecretPostgresPassword).Fields.Password).
+			To(Equal(primaryVault.GetSecret(files.SecretPostgresPassword).Fields.Password))
+		Expect(secondaryVault.GetSecret(files.SecretTokenPrivateKey).File.Content).
+			To(Equal(primaryVault.GetSecret(files.SecretTokenPrivateKey).File.Content))
+		Expect(secondaryVault.GetSecret("postgresPasswordAuth").Fields.Password).
+			To(Equal(primaryVault.GetSecret("postgresPasswordAuth").Fields.Password))
+
+		By("regenerating the per-cluster ingress CA and cephadm key")
+		Expect(secondaryVault.GetSecret(files.SecretSelfSignedCaKeyPem)).ToNot(BeNil())
+		Expect(secondaryVault.GetSecret(files.SecretSelfSignedCaKeyPem).File.Content).
+			ToNot(Equal(primaryVault.GetSecret(files.SecretSelfSignedCaKeyPem).File.Content))
+		Expect(secondaryVault.GetSecret(files.SecretCephSshPrivateKey)).ToNot(BeNil())
+		Expect(secondaryVault.GetSecret(files.SecretCephSshPrivateKey).File.Content).
+			ToNot(Equal(primaryVault.GetSecret(files.SecretCephSshPrivateKey).File.Content))
+
+		By("rewriting the config fields paired with the regenerated secrets")
+		Expect(secondaryCfg.Cluster.Certificates.CA.CertPem).ToNot(BeEmpty())
+		Expect(secondaryCfg.Cluster.Certificates.CA.CertPem).ToNot(Equal(primaryCfg.Cluster.Certificates.CA.CertPem))
+		Expect(secondaryCfg.Ceph.CephAdmSSHKey.PublicKey).ToNot(BeEmpty())
+		Expect(secondaryCfg.Ceph.CephAdmSSHKey.PublicKey).ToNot(Equal(primaryCfg.Ceph.CephAdmSSHKey.PublicKey))
+		Expect(secondaryCfg.Codesphere.CertIssuer.Acme.EABKeyID).To(BeEmpty())
+
+		By("not regenerating the shared postgres CA, since the server is external")
+		Expect(secondaryCfg.Postgres.CACertPem).To(Equal(primaryCfg.Postgres.CACertPem))
+	})
+})