From c5f430fb8f83b148bd328928c6e2726f77702a2c Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 15:14:11 +0200 Subject: [PATCH] refac(gcp): resolve the container registry once for all data centers The registry was written straight into the single install config, so which registry the nodes pull from was decided per config rather than per project. It now resolves onto the environment (ContainerRegistryURL plus credentials) and updateInstallConfig applies it to every data center's config and vault. All three registry types go through the same field, which also fixes the artifact registry never recording its URI on the create path. EnsureLocalContainerRegistry is split in two, because it early-returned when the registry was already running and thereby skipped distributing the registry certificate. A re-run that adds a data center hits exactly that path, and its nodes would then fail every pull with "certificate signed by unknown authority". Starting the registry stays conditional; distributing the certificate now always runs, over every data center's cluster nodes, which is safe because it is idempotent. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- internal/bootstrap/gcp/gcp.go | 64 +++++++++++++++------- internal/bootstrap/gcp/gcp_test.go | 69 +++++++++++++++++++----- internal/bootstrap/gcp/install_config.go | 12 ++++- 3 files changed, 112 insertions(+), 33 deletions(-) diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index d6a2d19f..632c28f7 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -711,7 +711,7 @@ func (b *GCPBootstrapper) EnsureArtifactRegistry() error { repo, err := b.GCPClient.GetArtifactRegistry(b.Env.ProjectID, b.Env.Region, repoName) if err == nil && repo != nil { - b.Env.InstallConfig.Registry.Server = repo.GetRegistryUri() + b.Env.ContainerRegistryURL = repo.GetRegistryUri() return nil } @@ -719,6 +719,7 @@ func (b *GCPBootstrapper) EnsureArtifactRegistry() error { if err != nil || repo == nil { return fmt.Errorf("failed to create artifact registry: %w, repo: %v", err, repo) } + b.Env.ContainerRegistryURL = repo.GetRegistryUri() return nil } @@ -1043,33 +1044,52 @@ func (b *GCPBootstrapper) EnsureHostsConfigured() error { return nil } -// EnsureLocalContainerRegistry installs a docker registry on the postgres node to speed up image loading time +// EnsureLocalContainerRegistry installs a container registry on the postgres node to speed up +// image loading time, and makes every cluster node of every data center trust its certificate. func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { + b.ensureDataCenters() + + registryServer, err := b.ensureRegistryRunning() + if err != nil { + return err + } + b.Env.ContainerRegistryURL = registryServer + + // The certificate must be distributed on every run, not only when the registry was just + // created: a re-run that adds a data center finds the registry already up, and that data + // center's nodes would otherwise not trust it. + return b.distributeRegistryCert(b.clusterNodes()) +} + +// ensureRegistryRunning starts the container registry on the postgres node and generates its +// credentials when it is not already serving. Returns the registry server address. +func (b *GCPBootstrapper) ensureRegistryRunning() (string, error) { localRegistryServer := b.Env.PostgreSQLNode.GetInternalIP() + ":5000" // Figure out if registry is already running b.stlog.Logf("Checking if local container registry is already running on postgres node") checkCommand := `test "$(podman ps --filter 'name=registry' --format '{{.Names}}' | wc -l)" -eq "1"` err := b.Env.PostgreSQLNode.RunSSHCommand("root", checkCommand) + vault := b.primaryDC().ConfigManager().GetVault() registryUsername := "" registryPassword := "" - if s := b.icg.GetVault().GetSecret(files.SecretRegistryUsername); s != nil && s.Fields != nil { + if s := vault.GetSecret(files.SecretRegistryUsername); s != nil && s.Fields != nil { registryUsername = s.Fields.Password } - if s := b.icg.GetVault().GetSecret(files.SecretRegistryPassword); s != nil && s.Fields != nil { + if s := vault.GetSecret(files.SecretRegistryPassword); s != nil && s.Fields != nil { registryPassword = s.Fields.Password } - if err == nil && b.Env.InstallConfig.Registry != nil && b.Env.InstallConfig.Registry.Server == localRegistryServer && - registryUsername != "" && registryPassword != "" { + if err == nil && registryUsername != "" && registryPassword != "" { b.stlog.Logf("Local container registry already running on postgres node") - return nil + b.Env.RegistryUsername = registryUsername + b.Env.RegistryPassword = registryPassword + return localRegistryServer, nil } - b.Env.InstallConfig.Registry.Server = localRegistryServer registryUsername = "custom-registry" registryPassword = shortuuid.New() - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: registryUsername}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: registryPassword}}) + b.Env.RegistryUsername = registryUsername + b.Env.RegistryPassword = registryPassword commands := []string{ "apt-get update", @@ -1089,19 +1109,24 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { -v /root/registry.crt:/certs/registry.crt \ -v /root/registry.key:/certs/registry.key \ registry:2`, - `mkdir -p /etc/docker/certs.d/` + b.Env.InstallConfig.Registry.Server, - `cp /root/registry.crt /etc/docker/certs.d/` + b.Env.InstallConfig.Registry.Server + `/ca.crt`, + `mkdir -p /etc/docker/certs.d/` + localRegistryServer, + `cp /root/registry.crt /etc/docker/certs.d/` + localRegistryServer + `/ca.crt`, } for _, cmd := range commands { b.stlog.Logf("Running command on postgres node: %s", util.Truncate(cmd, 12)) err := b.Env.PostgreSQLNode.RunSSHCommand("root", cmd) if err != nil { - return fmt.Errorf("failed to run command on postgres node: %w", err) + return "", fmt.Errorf("failed to run command on postgres node: %w", err) } } - allNodes := append(b.Env.ControlPlaneNodes, b.Env.CephNodes...) - for _, node := range allNodes { + return localRegistryServer, nil +} + +// distributeRegistryCert installs the local registry's self-signed certificate on the given +// nodes. It is idempotent, so it is safe — and required — to re-run for an additional data center. +func (b *GCPBootstrapper) distributeRegistryCert(nodes []*node.Node) error { + for _, node := range nodes { b.stlog.Logf("Configuring node '%s' to trust local registry certificate", node.GetName()) err := b.Env.PostgreSQLNode.RunSSHCommand("root", "scp -o StrictHostKeyChecking=no /root/registry.crt root@"+node.GetInternalIP()+":/usr/local/share/ca-certificates/registry.crt") if err != nil { @@ -1120,15 +1145,14 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { return nil } +// EnsureGitHubAccessConfigured resolves ghcr.io as the registry all data centers pull from. func (b *GCPBootstrapper) EnsureGitHubAccessConfigured() error { if b.Env.GitHubPAT == "" { return fmt.Errorf("GitHub PAT is not set") } - b.Env.InstallConfig.Registry.Server = "ghcr.io" - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: b.Env.RegistryUser}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: b.Env.GitHubPAT}}) - b.Env.InstallConfig.Registry.ReplaceImagesInBom = false - b.Env.InstallConfig.Registry.LoadContainerImages = false + b.Env.ContainerRegistryURL = "ghcr.io" + b.Env.RegistryUsername = b.Env.RegistryUser + b.Env.RegistryPassword = b.Env.GitHubPAT return nil } diff --git a/internal/bootstrap/gcp/gcp_test.go b/internal/bootstrap/gcp/gcp_test.go index a47e8754..0ba80054 100644 --- a/internal/bootstrap/gcp/gcp_test.go +++ b/internal/bootstrap/gcp/gcp_test.go @@ -851,8 +851,7 @@ var _ = Describe("GCP Bootstrapper", func() { Describe("EnsureLocalContainerRegistry", func() { Describe("Valid EnsureLocalContainerRegistry", func() { It("installs local registry", func() { - vault := &files.InstallVault{} - icg.EXPECT().GetVault().Return(vault) + icg.EXPECT().GetVault().Return(&files.InstallVault{}) // Setup mocked node // Check if running - return error to simulate not running @@ -868,7 +867,56 @@ var _ = Describe("GCP Bootstrapper", func() { err := bs.EnsureLocalContainerRegistry() Expect(err).NotTo(HaveOccurred()) - Expect(vault.GetSecret(files.SecretRegistryUsername).Fields.Password).To(Equal("custom-registry")) + Expect(bs.Env.RegistryUsername).To(Equal("custom-registry")) + Expect(bs.Env.RegistryPassword).NotTo(BeEmpty()) + Expect(bs.Env.ContainerRegistryURL).To(Equal(bs.Env.PostgreSQLNode.GetInternalIP() + ":5000")) + }) + + // A re-run that adds a data center finds the registry already up. Its nodes still + // need the registry's self-signed certificate, or every image pull fails. + It("distributes the registry certificate even when the registry is already running", func() { + vault := &files.InstallVault{} + vault.SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: "custom-registry"}}) + vault.SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: "existing-password"}}) + icg.EXPECT().GetVault().Return(vault) + + bs.Env.MultiDC = true + bs.Env.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1", nodeClient)} + bs.Env.CephNodes = []*node.Node{fakeNode("ceph-1", nodeClient)} + secondary := &gcp.DataCenter{ID: 2, Suffix: "-dc2"} + secondary.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1-dc2", nodeClient)} + secondary.CephNodes = []*node.Node{fakeNode("ceph-1-dc2", nodeClient)} + + // Registry is already running with credentials in the vault. + nodeClient.EXPECT().RunCommand(bs.Env.PostgreSQLNode, "root", mock.MatchedBy(func(cmd string) bool { + return strings.Contains(cmd, "podman ps") + })).Return(nil) + + scpTargets := []string{} + nodeClient.EXPECT().RunCommand(bs.Env.PostgreSQLNode, "root", mock.MatchedBy(func(cmd string) bool { + return strings.HasPrefix(cmd, "scp ") + })).RunAndReturn(func(_ *node.Node, _ string, cmd string) error { + scpTargets = append(scpTargets, cmd) + return nil + }).Times(4) + nodeClient.EXPECT().RunCommand(mock.Anything, "root", "update-ca-certificates").Return(nil).Times(4) + nodeClient.EXPECT().RunCommand(mock.Anything, "root", "systemctl restart docker.service || true").Return(nil).Times(4) + + // Register the second data center only after ensureDataCenters would have run, + // mirroring what EnsureComputeInstances produces for a --multi-dc bootstrap. + bs.Env.DataCenters = []*gcp.DataCenter{ + { + ID: 1, + ControlPlaneNodes: bs.Env.ControlPlaneNodes, + CephNodes: bs.Env.CephNodes, + }, + secondary, + } + bs.Env.DataCenters[0].SetConfigManager(icg) + + Expect(bs.EnsureLocalContainerRegistry()).To(Succeed()) + Expect(scpTargets).To(HaveLen(4)) + Expect(bs.Env.RegistryPassword).To(Equal("existing-password")) }) }) @@ -988,17 +1036,14 @@ var _ = Describe("GCP Bootstrapper", func() { csEnv.GitHubPAT = "fake-pat" csEnv.RegistryUser = "custom-registry" }) - It("sets configuration options in installconfig", func() { - vault := &files.InstallVault{} - icg.EXPECT().GetVault().Return(vault) - + // The resolved registry and its credentials live on the environment; every data center's + // config picks them up in updateInstallConfig. + It("resolves ghcr.io as the registry for all data centers", func() { err := bs.EnsureGitHubAccessConfigured() Expect(err).NotTo(HaveOccurred()) - Expect(bs.Env.InstallConfig.Registry.Server).To(Equal("ghcr.io")) - Expect(vault.GetSecret(files.SecretRegistryUsername).Fields.Password).To(Equal(csEnv.RegistryUser)) - Expect(vault.GetSecret(files.SecretRegistryPassword).Fields.Password).To(Equal(csEnv.GitHubPAT)) - Expect(bs.Env.InstallConfig.Registry.LoadContainerImages).To(BeFalse()) - Expect(bs.Env.InstallConfig.Registry.ReplaceImagesInBom).To(BeFalse()) + Expect(bs.Env.ContainerRegistryURL).To(Equal("ghcr.io")) + Expect(bs.Env.RegistryUsername).To(Equal(csEnv.RegistryUser)) + Expect(bs.Env.RegistryPassword).To(Equal(csEnv.GitHubPAT)) }) Context("When GitHub PAT is missing", func() { diff --git a/internal/bootstrap/gcp/install_config.go b/internal/bootstrap/gcp/install_config.go index 4a78fb32..4c80f872 100644 --- a/internal/bootstrap/gcp/install_config.go +++ b/internal/bootstrap/gcp/install_config.go @@ -141,7 +141,17 @@ func (b *GCPBootstrapper) updateInstallConfig(dc *DataCenter) error { // secrets.baseDir, so sharing a directory would let one data center's ceph and kubernetes // steps overwrite another's credentials. dc.InstallConfig.Secrets.BaseDir = dc.SecretsDir - if b.Env.RegistryType != RegistryTypeGitHub { + if b.Env.ContainerRegistryURL != "" { + dc.InstallConfig.Registry.Server = b.Env.ContainerRegistryURL + } + if b.Env.RegistryUsername != "" || b.Env.RegistryPassword != "" { + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: b.Env.RegistryUsername}}) + dc.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: b.Env.RegistryPassword}}) + } + if b.Env.RegistryType == RegistryTypeGitHub { + dc.InstallConfig.Registry.ReplaceImagesInBom = false + dc.InstallConfig.Registry.LoadContainerImages = false + } else { dc.InstallConfig.Registry.ReplaceImagesInBom = true dc.InstallConfig.Registry.LoadContainerImages = true }