refac(gcp): resolve the container registry once for all data centers

internal/bootstrap/gcp/gcp.go

@@ -711,14 +711,15 @@ func (b *GCPBootstrapper) EnsureArtifactRegistry() error {
 
 	repo, err := b.GCPClient.GetArtifactRegistry(b.Env.ProjectID, b.Env.Region, repoName)
 	if err == nil && repo != nil {
-		b.Env.InstallConfig.Registry.Server = repo.GetRegistryUri()
+		b.Env.ContainerRegistryURL = repo.GetRegistryUri()
 		return nil
 	}
 
 	repo, err = b.GCPClient.CreateArtifactRegistry(b.Env.ProjectID, b.Env.Region, repoName)
 	if err != nil || repo == nil {
 		return fmt.Errorf("failed to create artifact registry: %w, repo: %v", err, repo)
 	}
+	b.Env.ContainerRegistryURL = repo.GetRegistryUri()
 
 	return nil
 }
@@ -1043,33 +1044,52 @@ func (b *GCPBootstrapper) EnsureHostsConfigured() error {
 	return nil
 }
 
-// EnsureLocalContainerRegistry installs a docker registry on the postgres node to speed up image loading time
+// EnsureLocalContainerRegistry installs a container registry on the postgres node to speed up
+// image loading time, and makes every cluster node of every data center trust its certificate.
 func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error {
+	b.ensureDataCenters()
+
+	registryServer, err := b.ensureRegistryRunning()
+	if err != nil {
+		return err
+	}
+	b.Env.ContainerRegistryURL = registryServer
+
+	// The certificate must be distributed on every run, not only when the registry was just
+	// created: a re-run that adds a data center finds the registry already up, and that data
+	// center's nodes would otherwise not trust it.
+	return b.distributeRegistryCert(b.clusterNodes())
+}
+
+// ensureRegistryRunning starts the container registry on the postgres node and generates its
+// credentials when it is not already serving. Returns the registry server address.
+func (b *GCPBootstrapper) ensureRegistryRunning() (string, error) {
 	localRegistryServer := b.Env.PostgreSQLNode.GetInternalIP() + ":5000"
 
 	// Figure out if registry is already running
 	b.stlog.Logf("Checking if local container registry is already running on postgres node")
 	checkCommand := `test "$(podman ps --filter 'name=registry' --format '{{.Names}}' | wc -l)" -eq "1"`
 	err := b.Env.PostgreSQLNode.RunSSHCommand("root", checkCommand)
+	vault := b.primaryDC().ConfigManager().GetVault()
 	registryUsername := ""
 	registryPassword := ""
-	if s := b.icg.GetVault().GetSecret(files.SecretRegistryUsername); s != nil && s.Fields != nil {
+	if s := vault.GetSecret(files.SecretRegistryUsername); s != nil && s.Fields != nil {
 		registryUsername = s.Fields.Password
 	}
-	if s := b.icg.GetVault().GetSecret(files.SecretRegistryPassword); s != nil && s.Fields != nil {
+	if s := vault.GetSecret(files.SecretRegistryPassword); s != nil && s.Fields != nil {
 		registryPassword = s.Fields.Password
 	}
-	if err == nil && b.Env.InstallConfig.Registry != nil && b.Env.InstallConfig.Registry.Server == localRegistryServer &&
-		registryUsername != "" && registryPassword != "" {
+	if err == nil && registryUsername != "" && registryPassword != "" {
 		b.stlog.Logf("Local container registry already running on postgres node")
-		return nil
+		b.Env.RegistryUsername = registryUsername
+		b.Env.RegistryPassword = registryPassword
+		return localRegistryServer, nil
 	}
 
-	b.Env.InstallConfig.Registry.Server = localRegistryServer
 	registryUsername = "custom-registry"
 	registryPassword = shortuuid.New()
-	b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: registryUsername}})
-	b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: registryPassword}})
+	b.Env.RegistryUsername = registryUsername
+	b.Env.RegistryPassword = registryPassword
 
 	commands := []string{
 		"apt-get update",
@@ -1089,19 +1109,24 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error {
 		-v /root/registry.crt:/certs/registry.crt \
 		-v /root/registry.key:/certs/registry.key \
 		registry:2`,
-		`mkdir -p /etc/docker/certs.d/` + b.Env.InstallConfig.Registry.Server,
-		`cp /root/registry.crt /etc/docker/certs.d/` + b.Env.InstallConfig.Registry.Server + `/ca.crt`,
+		`mkdir -p /etc/docker/certs.d/` + localRegistryServer,
+		`cp /root/registry.crt /etc/docker/certs.d/` + localRegistryServer + `/ca.crt`,
 	}
 	for _, cmd := range commands {
 		b.stlog.Logf("Running command on postgres node: %s", util.Truncate(cmd, 12))
 		err := b.Env.PostgreSQLNode.RunSSHCommand("root", cmd)
 		if err != nil {
-			return fmt.Errorf("failed to run command on postgres node: %w", err)
+			return "", fmt.Errorf("failed to run command on postgres node: %w", err)
 		}
 	}
 
-	allNodes := append(b.Env.ControlPlaneNodes, b.Env.CephNodes...)
-	for _, node := range allNodes {
+	return localRegistryServer, nil
+}
+
+// distributeRegistryCert installs the local registry's self-signed certificate on the given
+// nodes. It is idempotent, so it is safe — and required — to re-run for an additional data center.
+func (b *GCPBootstrapper) distributeRegistryCert(nodes []*node.Node) error {
+	for _, node := range nodes {
 		b.stlog.Logf("Configuring node '%s' to trust local registry certificate", node.GetName())
 		err := b.Env.PostgreSQLNode.RunSSHCommand("root", "scp -o StrictHostKeyChecking=no /root/registry.crt root@"+node.GetInternalIP()+":/usr/local/share/ca-certificates/registry.crt")
 		if err != nil {
@@ -1120,15 +1145,14 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error {
 	return nil
 }
 
+// EnsureGitHubAccessConfigured resolves ghcr.io as the registry all data centers pull from.
 func (b *GCPBootstrapper) EnsureGitHubAccessConfigured() error {
 	if b.Env.GitHubPAT == "" {
 		return fmt.Errorf("GitHub PAT is not set")
 	}
-	b.Env.InstallConfig.Registry.Server = "ghcr.io"
-	b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: b.Env.RegistryUser}})
-	b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: b.Env.GitHubPAT}})
-	b.Env.InstallConfig.Registry.ReplaceImagesInBom = false
-	b.Env.InstallConfig.Registry.LoadContainerImages = false
+	b.Env.ContainerRegistryURL = "ghcr.io"
+	b.Env.RegistryUsername = b.Env.RegistryUser
+	b.Env.RegistryPassword = b.Env.GitHubPAT
 	return nil
 }
 

internal/bootstrap/gcp/gcp_test.go

@@ -851,8 +851,7 @@ var _ = Describe("GCP Bootstrapper", func() {
 	Describe("EnsureLocalContainerRegistry", func() {
 		Describe("Valid EnsureLocalContainerRegistry", func() {
 			It("installs local registry", func() {
-				vault := &files.InstallVault{}
-				icg.EXPECT().GetVault().Return(vault)
+				icg.EXPECT().GetVault().Return(&files.InstallVault{})
 
 				// Setup mocked node
 				// Check if running - return error to simulate not running
@@ -868,7 +867,56 @@ var _ = Describe("GCP Bootstrapper", func() {
 
 				err := bs.EnsureLocalContainerRegistry()
 				Expect(err).NotTo(HaveOccurred())
-				Expect(vault.GetSecret(files.SecretRegistryUsername).Fields.Password).To(Equal("custom-registry"))
+				Expect(bs.Env.RegistryUsername).To(Equal("custom-registry"))
+				Expect(bs.Env.RegistryPassword).NotTo(BeEmpty())
+				Expect(bs.Env.ContainerRegistryURL).To(Equal(bs.Env.PostgreSQLNode.GetInternalIP() + ":5000"))
+			})
+
+			// A re-run that adds a data center finds the registry already up. Its nodes still
+			// need the registry's self-signed certificate, or every image pull fails.
+			It("distributes the registry certificate even when the registry is already running", func() {
+				vault := &files.InstallVault{}
+				vault.SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: "custom-registry"}})
+				vault.SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: "existing-password"}})
+				icg.EXPECT().GetVault().Return(vault)
+
+				bs.Env.MultiDC = true
+				bs.Env.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1", nodeClient)}
+				bs.Env.CephNodes = []*node.Node{fakeNode("ceph-1", nodeClient)}
+				secondary := &gcp.DataCenter{ID: 2, Suffix: "-dc2"}
+				secondary.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1-dc2", nodeClient)}
+				secondary.CephNodes = []*node.Node{fakeNode("ceph-1-dc2", nodeClient)}
+
+				// Registry is already running with credentials in the vault.
+				nodeClient.EXPECT().RunCommand(bs.Env.PostgreSQLNode, "root", mock.MatchedBy(func(cmd string) bool {
+					return strings.Contains(cmd, "podman ps")
+				})).Return(nil)
+
+				scpTargets := []string{}
+				nodeClient.EXPECT().RunCommand(bs.Env.PostgreSQLNode, "root", mock.MatchedBy(func(cmd string) bool {
+					return strings.HasPrefix(cmd, "scp ")
+				})).RunAndReturn(func(_ *node.Node, _ string, cmd string) error {
+					scpTargets = append(scpTargets, cmd)
+					return nil
+				}).Times(4)
+				nodeClient.EXPECT().RunCommand(mock.Anything, "root", "update-ca-certificates").Return(nil).Times(4)
+				nodeClient.EXPECT().RunCommand(mock.Anything, "root", "systemctl restart docker.service || true").Return(nil).Times(4)
+
+				// Register the second data center only after ensureDataCenters would have run,
+				// mirroring what EnsureComputeInstances produces for a --multi-dc bootstrap.
+				bs.Env.DataCenters = []*gcp.DataCenter{
+					{
+						ID:                1,
+						ControlPlaneNodes: bs.Env.ControlPlaneNodes,
+						CephNodes:         bs.Env.CephNodes,
+					},
+					secondary,
+				}
+				bs.Env.DataCenters[0].SetConfigManager(icg)
+
+				Expect(bs.EnsureLocalContainerRegistry()).To(Succeed())
+				Expect(scpTargets).To(HaveLen(4))
+				Expect(bs.Env.RegistryPassword).To(Equal("existing-password"))
 			})
 		})
 
@@ -988,17 +1036,14 @@ var _ = Describe("GCP Bootstrapper", func() {
 			csEnv.GitHubPAT = "fake-pat"
 			csEnv.RegistryUser = "custom-registry"
 		})
-		It("sets configuration options in installconfig", func() {
-			vault := &files.InstallVault{}
-			icg.EXPECT().GetVault().Return(vault)
-
+		// The resolved registry and its credentials live on the environment; every data center's
+		// config picks them up in updateInstallConfig.
+		It("resolves ghcr.io as the registry for all data centers", func() {
 			err := bs.EnsureGitHubAccessConfigured()
 			Expect(err).NotTo(HaveOccurred())
-			Expect(bs.Env.InstallConfig.Registry.Server).To(Equal("ghcr.io"))
-			Expect(vault.GetSecret(files.SecretRegistryUsername).Fields.Password).To(Equal(csEnv.RegistryUser))
-			Expect(vault.GetSecret(files.SecretRegistryPassword).Fields.Password).To(Equal(csEnv.GitHubPAT))
-			Expect(bs.Env.InstallConfig.Registry.LoadContainerImages).To(BeFalse())
-			Expect(bs.Env.InstallConfig.Registry.ReplaceImagesInBom).To(BeFalse())
+			Expect(bs.Env.ContainerRegistryURL).To(Equal("ghcr.io"))
+			Expect(bs.Env.RegistryUsername).To(Equal(csEnv.RegistryUser))
+			Expect(bs.Env.RegistryPassword).To(Equal(csEnv.GitHubPAT))
 		})
 
 		Context("When GitHub PAT is missing", func() {

internal/bootstrap/gcp/install_config.go

@@ -141,7 +141,17 @@ func (b *GCPBootstrapper) updateInstallConfig(dc *DataCenter) error {
 	// secrets.baseDir, so sharing a directory would let one data center's ceph and kubernetes
 	// steps overwrite another's credentials.
 	dc.InstallConfig.Secrets.BaseDir = dc.SecretsDir
-	if b.Env.RegistryType != RegistryTypeGitHub {
+	if b.Env.ContainerRegistryURL != "" {
+		dc.InstallConfig.Registry.Server = b.Env.ContainerRegistryURL
+	}
+	if b.Env.RegistryUsername != "" || b.Env.RegistryPassword != "" {
+		dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: b.Env.RegistryUsername}})
+		dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: b.Env.RegistryPassword}})
+	}
+	if b.Env.RegistryType == RegistryTypeGitHub {
+		dc.InstallConfig.Registry.ReplaceImagesInBom = false
+		dc.InstallConfig.Registry.LoadContainerImages = false
+	} else {
 		dc.InstallConfig.Registry.ReplaceImagesInBom = true
 		dc.InstallConfig.Registry.LoadContainerImages = true
 	}