From d1de6c993ef0afe6701a216da47b66010c9e1177 Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 15:06:24 +0200 Subject: [PATCH] fix(gcp): give every data center its own DNS records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workspaces resolve per data center, and so does the platform: the frontend asks . for the configuration of the data center a workspace lives in. OMS only created cs. and its wildcard, both pointing at the first data center's gateway, so with more than one data center the second one's endpoint resolved to the first's gateway, which has no route for that host — every browser request for it was reset, and since the frontend fetches that config before rendering, the whole UI failed. EnsureDNSRecords now creates, per data center, its workspace hosting names and SSH proxy name pointing at its own public gateway and SSH proxy, plus .cs. and its wildcard pointing at its own platform gateway. The per-data-center platform names are only created when there is more than one data center: a single one is the primary, which cs. already resolves to. The records that were created are recorded in the infra file, so cleanup deletes exactly those. DeleteDNSRecordSets therefore takes the record list instead of a base domain, and cleanup falls back to deriving the names for infra files written before this and for a cleanup driven only by --project-id. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- cli/cmd/bootstrap_gcp_cleanup_test.go | 33 +++++- internal/bootstrap/gcp/cleanup.go | 15 ++- internal/bootstrap/gcp/datacenter.go | 10 ++ internal/bootstrap/gcp/datacenter_test.go | 27 +++++ internal/bootstrap/gcp/gcp.go | 116 ++++++++++++++-------- internal/bootstrap/gcp/gcp_client.go | 8 +- internal/bootstrap/gcp/gcp_test.go | 35 +++++++ internal/bootstrap/gcp/mocks.go | 22 ++-- 8 files changed, 210 insertions(+), 56 deletions(-) diff --git a/cli/cmd/bootstrap_gcp_cleanup_test.go b/cli/cmd/bootstrap_gcp_cleanup_test.go index 3337fd4c..75c4da29 100644 --- a/cli/cmd/bootstrap_gcp_cleanup_test.go +++ b/cli/cmd/bootstrap_gcp_cleanup_test.go @@ -322,7 +322,38 @@ var _ = Describe("BootstrapGcpCleanupCmd", func() { mockFileIO.EXPECT().Exists("/tmp/test-infra.json").Return(true) mockFileIO.EXPECT().ReadFile("/tmp/test-infra.json").Return(envData, nil) - mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", "example.com").Return(nil) + // An infra file without a recorded record list predates multi-DC support, so + // cleanup falls back to the single-data-center record names. + mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", gcp.GetDNSRecordNames("example.com")).Return(nil) + mockGCPClient.EXPECT().DeleteProject("test-project").Return(nil) + mockFileIO.EXPECT().Remove("/tmp/test-infra.json").Return(nil) + + err := cleanupCmd.ExecuteCleanup(deps) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("when the infra file recorded the DNS records it created", func() { + It("should delete exactly those records", func() { + cleanupCmd.Opts.ProjectID = "test-project" + cleanupCmd.Opts.Force = true + + recorded := []gcp.DNSRecordName{ + {Name: "cs.example.com.", Rtype: "A"}, + {Name: "2.ws.example.com.", Rtype: "A"}, + } + validEnv := gcp.CodesphereEnvironment{ + ProjectID: "test-project", + BaseDomain: "example.com", + DNSZoneName: "test-zone", + MultiDC: true, + DNSRecords: recorded, + } + envData, _ := json.Marshal(validEnv) + + mockFileIO.EXPECT().Exists("/tmp/test-infra.json").Return(true) + mockFileIO.EXPECT().ReadFile("/tmp/test-infra.json").Return(envData, nil) + mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", recorded).Return(nil) mockGCPClient.EXPECT().DeleteProject("test-project").Return(nil) mockFileIO.EXPECT().Remove("/tmp/test-infra.json").Return(nil) diff --git a/internal/bootstrap/gcp/cleanup.go b/internal/bootstrap/gcp/cleanup.go index f30e59a7..f16f3885 100644 --- a/internal/bootstrap/gcp/cleanup.go +++ b/internal/bootstrap/gcp/cleanup.go @@ -180,7 +180,20 @@ func (e *CleanupExecutor) CleanupDNSRecords() error { log.Printf("Skipping DNS cleanup: missing base domain or DNS zone name (provide --base-domain/--dns-zone-name or use --skip-dns-cleanup)") return nil } - return e.Deps.GCPClient.DeleteDNSRecordSets(e.DNSProjectID, e.DNSZoneName, e.BaseDomain) + return e.Deps.GCPClient.DeleteDNSRecordSets(e.DNSProjectID, e.DNSZoneName, e.dnsRecords()) +} + +// dnsRecords returns the DNS records to delete. The bootstrap records what it created in the +// infra file, which is authoritative. Older infra files predate that, and a cleanup driven only +// by --project-id has no infra file at all, so both fall back to deriving the names. +func (e *CleanupExecutor) dnsRecords() []DNSRecordName { + if len(e.InfraEnv.DNSRecords) > 0 { + return e.InfraEnv.DNSRecords + } + if len(e.InfraEnv.DataCenters) > 0 { + return DataCenterDNSRecordNames(e.BaseDomain, e.InfraEnv.DataCenters) + } + return GetDNSRecordNames(e.BaseDomain) } // RemoveDNSIAMBinding removes the cloud-controller service account's IAM binding diff --git a/internal/bootstrap/gcp/datacenter.go b/internal/bootstrap/gcp/datacenter.go index 7ef4fbcc..d263137f 100644 --- a/internal/bootstrap/gcp/datacenter.go +++ b/internal/bootstrap/gcp/datacenter.go @@ -82,6 +82,16 @@ func (dc *DataCenter) RemoteAgeKeyPath() string { return filepath.Join(dc.SecretsDir, "age_key.txt") } +// PlatformDomain returns the host this data center serves its own platform services on. The +// platform derives it from the data center ID and codesphere.domain, which OMS sets to +// cs., and the frontend calls it directly — the browser asks +// .cs. for the config of the data center a workspace lives in. So it has to +// resolve to this data center's platform gateway, not to the primary's, which is what +// cs. and its wildcard point at. +func (dc *DataCenter) PlatformDomain(baseDomain string) string { + return fmt.Sprintf("%d.cs.%s", dc.ID, baseDomain) +} + // K0sConfigScriptPath returns the local filename of this data center's k0s configuration script. func (dc *DataCenter) K0sConfigScriptPath() string { return fmt.Sprintf("configure-k0s%s.sh", dc.Suffix) diff --git a/internal/bootstrap/gcp/datacenter_test.go b/internal/bootstrap/gcp/datacenter_test.go index 76522bbf..f822dbb4 100644 --- a/internal/bootstrap/gcp/datacenter_test.go +++ b/internal/bootstrap/gcp/datacenter_test.go @@ -103,3 +103,30 @@ var _ = Describe("BuildDataCenters", func() { Expect(gcp.BuildDataCenters(env, installer.NewInstallConfigManager)[0].Name).To(Equal("dev")) }) }) + +var _ = Describe("DataCenterDNSRecordNames", func() { + It("returns the single-DC records for one data center", func() { + dcs := gcp.BuildDataCenters(&gcp.CodesphereEnvironment{BaseDomain: "example.com"}, nil) + + Expect(gcp.DataCenterDNSRecordNames("example.com", dcs)).To(ConsistOf(gcp.GetDNSRecordNames("example.com"))) + }) + + It("shares the platform names and scopes the workspace names per data center", func() { + dcs := gcp.BuildDataCenters(&gcp.CodesphereEnvironment{MultiDC: true, BaseDomain: "example.com"}, nil) + + Expect(gcp.DataCenterDNSRecordNames("example.com", dcs)).To(Equal([]gcp.DNSRecordName{ + {Name: "cs.example.com.", Rtype: "A"}, + {Name: "*.cs.example.com.", Rtype: "A"}, + {Name: "1.ws.example.com.", Rtype: "A"}, + {Name: "*.1.ws.example.com.", Rtype: "A"}, + {Name: "*.1.ssh.cs.example.com.", Rtype: "A"}, + {Name: "1.cs.example.com.", Rtype: "A"}, + {Name: "*.1.cs.example.com.", Rtype: "A"}, + {Name: "2.ws.example.com.", Rtype: "A"}, + {Name: "*.2.ws.example.com.", Rtype: "A"}, + {Name: "*.2.ssh.cs.example.com.", Rtype: "A"}, + {Name: "2.cs.example.com.", Rtype: "A"}, + {Name: "*.2.cs.example.com.", Rtype: "A"}, + })) + }) +}) diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index 72aaa895..d6a2d19f 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -63,15 +63,17 @@ func CheckOMSManagedLabel(labels map[string]string) bool { return exists && value == "true" } -// GetDNSRecordNames returns the DNS record names that OMS creates for a given base domain. -func GetDNSRecordNames(baseDomain string) []struct { - Name string - Rtype string -} { - return []struct { - Name string - Rtype string - }{ +// DNSRecordName identifies a DNS record set that OMS manages. +type DNSRecordName struct { + Name string `json:"name"` + Rtype string `json:"rtype"` +} + +// GetDNSRecordNames returns the DNS record names a single-data-center bootstrap creates for a +// given base domain. It is the fallback for infra files written before multi-DC support, which +// do not record the created records. +func GetDNSRecordNames(baseDomain string) []DNSRecordName { + return []DNSRecordName{ {fmt.Sprintf("cs.%s.", baseDomain), "A"}, {fmt.Sprintf("*.cs.%s.", baseDomain), "A"}, {fmt.Sprintf("ws.%s.", baseDomain), "A"}, @@ -80,6 +82,30 @@ func GetDNSRecordNames(baseDomain string) []struct { } } +// DataCenterDNSRecordNames returns every DNS record OMS creates for the given data center +// layout: the shared platform gateway names plus each data center's workspace and SSH names. +func DataCenterDNSRecordNames(baseDomain string, dcs []*DataCenter) []DNSRecordName { + records := []DNSRecordName{ + {fmt.Sprintf("cs.%s.", baseDomain), "A"}, + {fmt.Sprintf("*.cs.%s.", baseDomain), "A"}, + } + for _, dc := range dcs { + records = append(records, + DNSRecordName{fmt.Sprintf("%s.", dc.WorkspaceHostingBaseDomain), "A"}, + DNSRecordName{fmt.Sprintf("*.%s.", dc.WorkspaceHostingBaseDomain), "A"}, + DNSRecordName{fmt.Sprintf("*.%s.", dc.SshBaseDomain), "A"}, + ) + if len(dcs) > 1 { + records = append(records, + DNSRecordName{fmt.Sprintf("%s.", dc.PlatformDomain(baseDomain)), "A"}, + DNSRecordName{fmt.Sprintf("*.%s.", dc.PlatformDomain(baseDomain)), "A"}, + ) + } + } + + return records +} + // This should ALWAYS be empty. Internal flags are for internal feature // development and not intended for customer use. // Atm. it's not empty as the internal flags below are likely preview or @@ -160,6 +186,9 @@ type CodesphereEnvironment struct { MultiDC bool `json:"multi_dc"` // DataCenters holds the per-data-center state. It always has at least one entry. DataCenters []*DataCenter `json:"datacenters"` + // DNSRecords records the DNS records the bootstrap created, so cleanup deletes exactly + // those instead of recomputing the list. + DNSRecords []DNSRecordName `json:"dns_records,omitempty"` // Mirrors of DataCenters[0], written for infra files consumed by cleanup and restart-vms. ControlPlaneNodes []*node.Node `json:"control_plane_nodes"` CephNodes []*node.Node `json:"ceph_nodes"` @@ -1104,6 +1133,8 @@ func (b *GCPBootstrapper) EnsureGitHubAccessConfigured() error { } func (b *GCPBootstrapper) EnsureDNSRecords() error { + b.ensureDataCenters() + gcpProject := b.Env.DNSProjectID if b.Env.DNSProjectID == "" { gcpProject = b.Env.ProjectID @@ -1115,37 +1146,30 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error { return fmt.Errorf("failed to ensure DNS managed zone: %w", err) } + // The platform is served from one domain shared by all data centers, pointing at the + // primary data center's gateway. records := []*dns.ResourceRecordSet{ - { - Name: fmt.Sprintf("cs.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.GatewayIP}, - }, - { - Name: fmt.Sprintf("*.cs.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.GatewayIP}, - }, - { - Name: fmt.Sprintf("*.ws.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.PublicGatewayIP}, - }, - { - Name: fmt.Sprintf("ws.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.PublicGatewayIP}, - }, - { - Name: fmt.Sprintf("*.ssh.cs.%s.", b.Env.BaseDomain), - Type: "A", - Ttl: 300, - Rrdatas: []string{b.Env.SshProxyIP}, - }, + dnsARecord(fmt.Sprintf("cs.%s.", b.Env.BaseDomain), b.primaryDC().GatewayIP), + dnsARecord(fmt.Sprintf("*.cs.%s.", b.Env.BaseDomain), b.primaryDC().GatewayIP), + } + // Workspaces and their SSH endpoints resolve per data center, so each one gets its own + // names pointing at its own public gateway and SSH proxy. + for _, dc := range b.Env.DataCenters { + records = append(records, + dnsARecord(fmt.Sprintf("%s.", dc.WorkspaceHostingBaseDomain), dc.PublicGatewayIP), + dnsARecord(fmt.Sprintf("*.%s.", dc.WorkspaceHostingBaseDomain), dc.PublicGatewayIP), + dnsARecord(fmt.Sprintf("*.%s.", dc.SshBaseDomain), dc.SshProxyIP), + ) + // The platform calls each data center's own services at .cs., which + // the wildcard above would send to the primary data center's gateway. A single data + // center is that primary, so it needs no record of its own. + if len(b.Env.DataCenters) > 1 { + platformDomain := dc.PlatformDomain(b.Env.BaseDomain) + records = append(records, + dnsARecord(fmt.Sprintf("%s.", platformDomain), dc.GatewayIP), + dnsARecord(fmt.Sprintf("*.%s.", platformDomain), dc.GatewayIP), + ) + } } err = b.GCPClient.EnsureDNSRecordSets(gcpProject, zoneName, records) @@ -1153,9 +1177,23 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error { return fmt.Errorf("failed to ensure DNS record sets: %w", err) } + // Record what was created so cleanup deletes exactly these records instead of recomputing + // the list from the base domain. + b.Env.DNSRecords = DataCenterDNSRecordNames(b.Env.BaseDomain, b.Env.DataCenters) + return nil } +// dnsARecord builds a short-TTL A record set, as used during initial setup. +func dnsARecord(name, ip string) *dns.ResourceRecordSet { + return &dns.ResourceRecordSet{ + Name: name, + Type: "A", + Ttl: 300, + Rrdatas: []string{ip}, + } +} + // InstallCodesphere installs Codesphere into every data center from the shared jumpbox, in // ascending data center order. The order matters: the primary data center's install creates the // database, roles and schema that the secondary ones reuse. diff --git a/internal/bootstrap/gcp/gcp_client.go b/internal/bootstrap/gcp/gcp_client.go index 7b764de3..cb7a9c94 100644 --- a/internal/bootstrap/gcp/gcp_client.go +++ b/internal/bootstrap/gcp/gcp_client.go @@ -61,7 +61,7 @@ type GCPClientManager interface { GetAddress(projectID, region, addressName string) (*computepb.Address, error) EnsureDNSManagedZone(projectID, zoneName, dnsName, description string) error EnsureDNSRecordSets(projectID, zoneName string, records []*dns.ResourceRecordSet) error - DeleteDNSRecordSets(projectID, zoneName, baseDomain string) error + DeleteDNSRecordSets(projectID, zoneName string, records []DNSRecordName) error CreatePublicCAExternalAccountKey(projectID string) (keyID, b64MacKey string, err error) } @@ -828,15 +828,15 @@ func (c *GCPClient) EnsureDNSRecordSets(projectID, zoneName string, records []*d return nil } -// DeleteDNSRecordSets deletes DNS record sets created by OMS for the given base domain. -func (c *GCPClient) DeleteDNSRecordSets(projectID, zoneName, baseDomain string) error { +// DeleteDNSRecordSets deletes the given DNS record sets, ignoring those that no longer exist. +func (c *GCPClient) DeleteDNSRecordSets(projectID, zoneName string, records []DNSRecordName) error { service, err := dns.NewService(c.ctx) if err != nil { return fmt.Errorf("failed to create DNS service: %w", err) } var deletions []*dns.ResourceRecordSet - for _, record := range GetDNSRecordNames(baseDomain) { + for _, record := range records { existing, err := service.ResourceRecordSets.Get(projectID, zoneName, record.Name, record.Rtype).Context(c.ctx).Do() if IsNotFoundError(err) { continue diff --git a/internal/bootstrap/gcp/gcp_test.go b/internal/bootstrap/gcp/gcp_test.go index 073ef283..a47e8754 100644 --- a/internal/bootstrap/gcp/gcp_test.go +++ b/internal/bootstrap/gcp/gcp_test.go @@ -1419,6 +1419,41 @@ var _ = Describe("GCP Bootstrapper", func() { err := bs.EnsureDNSRecords() Expect(err).NotTo(HaveOccurred()) }) + + It("points each data center's platform host at its own gateway", func() { + bs.Env.DataCenters = []*gcp.DataCenter{ + { + ID: 1, GatewayIP: "1.1.1.1", PublicGatewayIP: "1.1.1.2", SshProxyIP: "1.1.1.3", + WorkspaceHostingBaseDomain: "1.ws.example.com", SshBaseDomain: "1.ssh.cs.example.com", + }, + { + ID: 2, Suffix: "-dc2", GatewayIP: "2.2.2.1", PublicGatewayIP: "2.2.2.2", SshProxyIP: "2.2.2.3", + WorkspaceHostingBaseDomain: "2.ws.example.com", SshBaseDomain: "2.ssh.cs.example.com", + }, + } + + gc.EXPECT().EnsureDNSManagedZone(csEnv.DNSProjectID, csEnv.DNSZoneName, csEnv.BaseDomain+".", mock.Anything).Return(nil) + targets := map[string]string{} + gc.EXPECT().EnsureDNSRecordSets(csEnv.DNSProjectID, csEnv.DNSZoneName, mock.Anything). + RunAndReturn(func(_ string, _ string, records []*dns.ResourceRecordSet) error { + for _, r := range records { + targets[r.Name] = r.Rrdatas[0] + } + return nil + }) + + Expect(bs.EnsureDNSRecords()).To(Succeed()) + // The shared platform name stays on the primary data center's gateway, but the + // per-data-center platform host the frontend calls resolves to its own gateway. + Expect(targets["cs.example.com."]).To(Equal("1.1.1.1")) + Expect(targets["*.cs.example.com."]).To(Equal("1.1.1.1")) + Expect(targets["1.cs.example.com."]).To(Equal("1.1.1.1")) + Expect(targets["2.cs.example.com."]).To(Equal("2.2.2.1")) + Expect(targets["*.2.cs.example.com."]).To(Equal("2.2.2.1")) + // Workspaces and SSH keep pointing at the public gateway and SSH proxy. + Expect(targets["2.ws.example.com."]).To(Equal("2.2.2.2")) + Expect(targets["*.2.ssh.cs.example.com."]).To(Equal("2.2.2.3")) + }) }) Describe("Invalid cases", func() { diff --git a/internal/bootstrap/gcp/mocks.go b/internal/bootstrap/gcp/mocks.go index 5b371325..74b64038 100644 --- a/internal/bootstrap/gcp/mocks.go +++ b/internal/bootstrap/gcp/mocks.go @@ -796,16 +796,16 @@ func (_c *MockGCPClientManager_CreateVPC_Call) RunAndReturn(run func(projectID s } // DeleteDNSRecordSets provides a mock function for the type MockGCPClientManager -func (_mock *MockGCPClientManager) DeleteDNSRecordSets(projectID string, zoneName string, baseDomain string) error { - ret := _mock.Called(projectID, zoneName, baseDomain) +func (_mock *MockGCPClientManager) DeleteDNSRecordSets(projectID string, zoneName string, records []DNSRecordName) error { + ret := _mock.Called(projectID, zoneName, records) if len(ret) == 0 { panic("no return value specified for DeleteDNSRecordSets") } var r0 error - if returnFunc, ok := ret.Get(0).(func(string, string, string) error); ok { - r0 = returnFunc(projectID, zoneName, baseDomain) + if returnFunc, ok := ret.Get(0).(func(string, string, []DNSRecordName) error); ok { + r0 = returnFunc(projectID, zoneName, records) } else { r0 = ret.Error(0) } @@ -820,12 +820,12 @@ type MockGCPClientManager_DeleteDNSRecordSets_Call struct { // DeleteDNSRecordSets is a helper method to define mock.On call // - projectID string // - zoneName string -// - baseDomain string -func (_e *MockGCPClientManager_Expecter) DeleteDNSRecordSets(projectID any, zoneName any, baseDomain any) *MockGCPClientManager_DeleteDNSRecordSets_Call { - return &MockGCPClientManager_DeleteDNSRecordSets_Call{Call: _e.mock.On("DeleteDNSRecordSets", projectID, zoneName, baseDomain)} +// - records []DNSRecordName +func (_e *MockGCPClientManager_Expecter) DeleteDNSRecordSets(projectID any, zoneName any, records any) *MockGCPClientManager_DeleteDNSRecordSets_Call { + return &MockGCPClientManager_DeleteDNSRecordSets_Call{Call: _e.mock.On("DeleteDNSRecordSets", projectID, zoneName, records)} } -func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Run(run func(projectID string, zoneName string, baseDomain string)) *MockGCPClientManager_DeleteDNSRecordSets_Call { +func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Run(run func(projectID string, zoneName string, records []DNSRecordName)) *MockGCPClientManager_DeleteDNSRecordSets_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 string if args[0] != nil { @@ -835,9 +835,9 @@ func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Run(run func(projectID if args[1] != nil { arg1 = args[1].(string) } - var arg2 string + var arg2 []DNSRecordName if args[2] != nil { - arg2 = args[2].(string) + arg2 = args[2].([]DNSRecordName) } run( arg0, @@ -853,7 +853,7 @@ func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) Return(err error) *Mock return _c } -func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) RunAndReturn(run func(projectID string, zoneName string, baseDomain string) error) *MockGCPClientManager_DeleteDNSRecordSets_Call { +func (_c *MockGCPClientManager_DeleteDNSRecordSets_Call) RunAndReturn(run func(projectID string, zoneName string, records []DNSRecordName) error) *MockGCPClientManager_DeleteDNSRecordSets_Call { _c.Call.Return(run) return _c }