From a42a58126ea736366221f467a88e09959f235124 Mon Sep 17 00:00:00 2001 From: Quan Huynh Date: Sat, 31 Oct 2026 21:06:19 +0700 Subject: [PATCH] fix(kubernetes): keep investigations and workloads stable during sync --- pkg/controllers/kubernetes_admin.go | 16 +- pkg/controllers/kubernetes_admin_test.go | 20 + .../cached_authority_regression_test.go | 55 +- pkg/kubernetes/evidence.go | 40 +- .../evidence_review_regression_test.go | 119 +++ pkg/kubernetes/inventory_cache.go | 30 +- pkg/kubernetes/inventory_cache_test.go | 248 +++++ .../inventory_continuity_adversarial_test.go | 142 +++ pkg/kubernetes/operations.go | 123 ++- pkg/kubernetes/service_test.go | 14 +- pkg/kubernetes/triage.go | 26 +- .../inventory_continuity_integration_test.go | 949 ++++++++++++++++++ ui/src/lib/api.ts | 20 +- ui/src/lib/kubernetesApi.test.ts | 34 + ui/src/lib/kubernetesContinuity.test.ts | 55 + ui/src/lib/kubernetesContinuity.ts | 30 +- ...bernetesFocusedHistory.adversarial.test.ts | 103 ++ ui/src/lib/kubernetesFocusedHistory.test.ts | 94 ++ ui/src/lib/kubernetesFocusedHistory.ts | 89 ++ ui/src/lib/useKubernetesBreakLogs.ts | 14 +- ui/src/lib/useKubernetesCluster.tsx | 16 +- ui/src/pages/KubernetesBreakLogs.tsx | 20 +- ui/src/pages/KubernetesBreakTimeline.test.tsx | 70 +- ui/src/pages/KubernetesBreakTimeline.tsx | 10 +- ui/src/pages/KubernetesExplorerFlow.test.tsx | 128 ++- ui/src/pages/KubernetesExplorerViews.tsx | 76 +- ui/src/pages/KubernetesPage.test.tsx | 276 ++++- ui/src/pages/KubernetesPage.tsx | 110 +- ui/src/pages/KubernetesResourceDetail.css | 6 + ui/tests/e2e/kubernetes-overview.spec.ts | 648 +++++++++++- ui/tsconfig.app.tsbuildinfo | 2 +- 31 files changed, 3366 insertions(+), 217 deletions(-) create mode 100644 pkg/kubernetes/inventory_continuity_adversarial_test.go create mode 100644 tests/fakekube/inventory_continuity_integration_test.go create mode 100644 ui/src/lib/kubernetesFocusedHistory.adversarial.test.ts create mode 100644 ui/src/lib/kubernetesFocusedHistory.test.ts create mode 100644 ui/src/lib/kubernetesFocusedHistory.ts diff --git a/pkg/controllers/kubernetes_admin.go b/pkg/controllers/kubernetes_admin.go index 0cc14f61..403bf0c6 100644 --- a/pkg/controllers/kubernetes_admin.go +++ b/pkg/controllers/kubernetes_admin.go @@ -199,13 +199,6 @@ func connectorResolutionError(err error) error { return fmt.Errorf("%w: %w", kubernetes.ErrConnectorCredentialUnresolved, err) } -func (controller *KubernetesAdminController) authorizeInfrastructureView(ctx *fiber.Ctx) error { - if !callerCanViewInfrastructure(ctx) { - return refuseCallerInfrastructureView(ctx) - } - return ctx.Next() -} - // callerCanViewInfrastructure answers the permission itself and records the // caller's authorization, but does not write a response or advance the chain. A // gate that still has to look at the deployment asks this first, so the answer it @@ -484,7 +477,7 @@ func deltaMatchesNamespace(delta kubernetesindex.Delta, namespace string) bool { } func (controller *KubernetesAdminController) issues(ctx *fiber.Ctx) error { args := ctx.Context().QueryArgs() - for _, key := range []string{"broken", "include", "root_uid", "context"} { + for _, key := range []string{"broken", "include", "root_uid", "context", "occurrence_id", "center"} { if len(args.PeekMulti(key)) > 1 { return writeKubernetes(ctx, nil, kubernetes.ErrInvalidArguments) } @@ -506,6 +499,11 @@ func (controller *KubernetesAdminController) issues(ctx *fiber.Ctx) error { if args.Has("context") && contextMode != "persisted" { return writeKubernetes(ctx, nil, kubernetes.ErrInvalidArguments) } + occurrenceID := strings.Clone(ctx.Query("occurrence_id")) + center, centerErr := parseKubernetesTime(ctx.Query("center")) + if centerErr != nil || (args.Has("occurrence_id") || args.Has("center")) && contextMode != "persisted" || args.Has("occurrence_id") && occurrenceID == "" || args.Has("center") && center.IsZero() { + return writeKubernetes(ctx, nil, kubernetes.ErrInvalidArguments) + } if args.Has("root_uid") && (root == "" || include != "changes" && contextMode != "persisted") { return writeKubernetes(ctx, nil, kubernetes.ErrInvalidArguments) } @@ -519,7 +517,7 @@ func (controller *KubernetesAdminController) issues(ctx *fiber.Ctx) error { } value, err := requestKubernetesService(ctx).Issues(ctx.UserContext(), kubernetes.IssueOptions{ Since: since, Until: until, Namespace: ctx.Query("namespace"), Severity: ctx.Query("severity"), Kind: ctx.Query("kind"), Limit: queryInt(ctx, "limit"), Cursor: ctx.Query("cursor"), - Broken: broken, IncludeChanges: include == "changes", RootUID: root, Context: contextMode, + Broken: broken, IncludeChanges: include == "changes", RootUID: root, Context: contextMode, OccurrenceID: occurrenceID, Center: center, }) return writeKubernetes(ctx, value, err) } diff --git a/pkg/controllers/kubernetes_admin_test.go b/pkg/controllers/kubernetes_admin_test.go index fee096e8..112ef314 100644 --- a/pkg/controllers/kubernetes_admin_test.go +++ b/pkg/controllers/kubernetes_admin_test.go @@ -359,6 +359,26 @@ func TestKubernetesIssuesQueryValidationAndInvestigate(t *testing.T) { return ctx.Next() }) NewKubernetesAdminController(service).Register(app.Group("/api")) + for _, query := range []string{ + "?occurrence_id=selected", + "?center=2026-10-01T00:00:00Z", + "?context=persisted&root_uid=deployment-checkout&occurrence_id=", + "?context=persisted&root_uid=deployment-checkout&occurrence_id=bad%2Fid", + "?context=persisted&root_uid=deployment-checkout&occurrence_id=a&occurrence_id=b", + "?context=persisted&root_uid=deployment-checkout¢er=", + "?context=persisted&root_uid=deployment-checkout¢er=not-a-time", + "?context=persisted&root_uid=deployment-checkout¢er=2999-01-01T00:00:00Z", + "?context=persisted&root_uid=deployment-checkout¢er=2026-10-01T00:00:00Z¢er=2026-10-01T00:00:00Z", + } { + response, err := app.Test(httptest.NewRequest("GET", "/api/admin/kubernetes/issues"+query, nil), -1) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != http.StatusBadRequest { + t.Fatalf("invalid selector accepted: status=%d", response.StatusCode) + } + } for _, test := range []struct { query string want int diff --git a/pkg/kubernetes/cached_authority_regression_test.go b/pkg/kubernetes/cached_authority_regression_test.go index dade07c0..87f46cef 100644 --- a/pkg/kubernetes/cached_authority_regression_test.go +++ b/pkg/kubernetes/cached_authority_regression_test.go @@ -2,10 +2,12 @@ package kubernetes import ( "bytes" + "context" "encoding/json" "errors" "net/http" "net/http/httptest" + "strconv" "strings" "sync/atomic" "testing" @@ -15,6 +17,40 @@ import ( "github.com/VersusControl/versus-incident/tests/fakekube" ) +// Observe the actual durable head and journal writes, without changing their +// timing or guards. The LIST attempt provenance identifies which cycle wrote a +// checkpoint even if its journal commit completes after permission revocation. +type authorityObservedStore struct { + storage.Provider + storage.BlobCAS + storage.EvidenceStore + t *testing.T + mode *atomic.Int32 +} + +func (p *authorityObservedStore) CompareAndSwapBlob(key string, old, next []byte) (bool, error) { + won, err := p.BlobCAS.CompareAndSwapBlob(key, old, next) + if won && err == nil { + var a, b inventoryLeaseRecord + if json.Unmarshal(old, &a) == nil && json.Unmarshal(next, &b) == nil && b.Revision != "" && b.Revision != a.Revision { + raw, readErr := p.ReadBlob(key + "/data/" + b.DataOwner + "/" + strconv.Itoa(b.Slot)) + var generation inventoryGeneration + if readErr == nil && json.Unmarshal(raw, &generation) == nil { + p.t.Logf("durable head mode=%d revision=%s pod=%+v deployment=%+v", p.mode.Load(), b.Revision, generation.View.Kinds["Pod"], generation.View.Kinds["Deployment"]) + } + } + } + return won, err +} + +func (p *authorityObservedStore) CommitEvidence(ctx context.Context, scope storage.EvidenceScope, batch storage.EvidenceBatch) error { + err := p.EvidenceStore.CommitEvidence(ctx, scope, batch) + if err == nil { + p.t.Logf("durable journal mode=%d id=%s expected=%s at=%s", p.mode.Load(), batch.ID, batch.Expected, batch.At.Format(time.RFC3339Nano)) + } + return err +} + func TestAutonomousCachedCatalogClassifiesCurrentMaterialAuthority(t *testing.T) { for _, backend := range []string{"memory", "file"} { for _, scenario := range []string{"all", "multiple-kinds", "secret-only"} { @@ -84,6 +120,7 @@ func TestAutonomousCachedCatalogClassifiesCurrentMaterialAuthority(t *testing.T) t.Fatal(err) } } + provider = &authorityObservedStore{Provider: provider, BlobCAS: provider.(storage.BlobCAS), EvidenceStore: provider.(storage.EvidenceStore), t: t, mode: &mode} service.SetChangeStorage(provider) registry := NewServiceRegistry(service) defer registry.Close() @@ -123,11 +160,16 @@ func TestAutonomousCachedCatalogClassifiesCurrentMaterialAuthority(t *testing.T) t.Fatal(err) } beforeRaw, _ := json.Marshal(before) - checkpoint, err := store.EvidenceCheckpoint(t.Context(), evidenceScope) - if err != nil { - t.Fatal(err) - } + // Ready data is visible before its journal commit finishes. Pin the + // checkpoint belonging to this exact head before revoking permission; + // an authorized baseline commit is not a denied-cycle publication. + var checkpoint storage.EvidenceCheckpoint + waitUntil(t, 8*time.Second, "baseline data-head journal committed", func() bool { + checkpoint, err = store.EvidenceCheckpoint(t.Context(), evidenceScope) + return err == nil && before.EvidenceBatch != nil && checkpoint.ID == before.EvidenceBatch.ID + }) catalogs := catalogReads.Load() + t.Logf("baseline checkpoint=%s at=%s revision=%s revoke_at=%s", checkpoint.ID, checkpoint.At.Format(time.RFC3339Nano), record.Revision, time.Now().UTC().Format(time.RFC3339Nano)) if scenario == "all" { mode.Store(1) } else { @@ -135,6 +177,7 @@ func TestAutonomousCachedCatalogClassifiesCurrentMaterialAuthority(t *testing.T) } select { case <-materialSeen: + t.Logf("first material denial at=%s", time.Now().UTC().Format(time.RFC3339Nano)) case <-time.After(40 * time.Second): t.Fatal("normal material cadence never observed the changed permissions") } @@ -177,7 +220,9 @@ func TestAutonomousCachedCatalogClassifiesCurrentMaterialAuthority(t *testing.T) report, _ := health() workloads, workErr := service.Workloads(t.Context(), WorkloadListOptions{}) issues, issueErr := service.Issues(t.Context(), IssueOptions{}) - t.Fatalf("cached catalog bypassed current proof: denied=%d sync=%s failing=%d unreadable=%d Workloads=%d/%v Issues=%d/%v", materialDenials.Load(), report.State, report.FailingKinds, report.UnreadableKinds, len(workloads.Items), workErr, len(issues.Items), issueErr) + current, _ := store.EvidenceCheckpoint(t.Context(), evidenceScope) + latest, head, _ := readInventoryGeneration(provider, scope, "") + t.Fatalf("cached catalog bypassed current proof: denied=%d sync=%s failing=%d unreadable=%d Workloads=%d/%v Issues=%d/%v checkpoint=%s/%s revision=%s pod=%+v", materialDenials.Load(), report.State, report.FailingKinds, report.UnreadableKinds, len(workloads.Items), workErr, len(issues.Items), issueErr, current.ID, current.At.Format(time.RFC3339Nano), head.Revision, latest.View.Kinds["Pod"]) } pending, _ := health() if pending.State == "ready" { diff --git a/pkg/kubernetes/evidence.go b/pkg/kubernetes/evidence.go index efb06837..718c3be5 100644 --- a/pkg/kubernetes/evidence.go +++ b/pkg/kubernetes/evidence.go @@ -721,7 +721,11 @@ func (service *Service) historicalIssues(ctx context.Context, options IssueOptio } func (service *Service) persistedRootContext(ctx context.Context, options IssueOptions) (IssuePage, error) { - page := IssuePage{Items: []Issue{}, Totals: map[string]int{}, Broken: BrokenTotals{BySeverity: map[string]int{}}, Sync: SyncStatus{State: "ready"}} + page := IssuePage{Items: []Issue{}, Totals: map[string]int{}, Broken: BrokenTotals{BySeverity: map[string]int{}}, Sync: SyncStatus{State: "ready"}, SelectionStatus: "not_requested"} + selecting := options.OccurrenceID != "" || !options.Center.IsZero() + if selecting { + page.SelectionStatus = "not_found" + } // Search retained occurrences, not the selected evidence interval. A missing // live object, recovered issue, or empty selected interval cannot erase this // identity. Each candidate still passes the normal current authority gate. @@ -733,19 +737,45 @@ func (service *Service) persistedRootContext(ctx context.Context, options IssueO } page.Sync.Partial = page.Sync.Partial || history.Sync.Partial page.Partial = append(page.Partial, history.Partial...) - if len(history.Items) > 0 { - issue := history.Items[0] + for _, issue := range history.Items { if issue.Root.UID != options.RootUID || options.Namespace != "" && issue.Root.Namespace != options.Namespace { return page, ErrIncompleteInventory } - page.RootContext = &PersistedRootContext{Root: issue.Root, Occurrence: issue, Retained: true} - return page, nil + if page.RootContext == nil { + page.RootContext = &PersistedRootContext{Root: issue.Root, Occurrence: issue, Retained: true} + } + if !selecting { + return page, nil + } + if options.OccurrenceID != "" { + // ID is authoritative: center never permits a different occurrence. + if issue.OccurrenceID == options.OccurrenceID { + page.SelectedOccurrence, page.SelectionStatus = &issue, "resolved" + return page, nil + } + } else if issue.BrokeAt != nil && issue.BrokeAt.Equal(options.Center) { + if page.SelectionStatus == "resolved" { + page.SelectedOccurrence, page.SelectionStatus = nil, "ambiguous" + } else if page.SelectionStatus != "ambiguous" { + page.SelectedOccurrence, page.SelectionStatus = &issue, "resolved" + } + } } if history.Next == "" { + if selecting && page.Sync.Partial && page.SelectionStatus != "ambiguous" { + page.SelectedOccurrence, page.SelectionStatus = nil, "partial" + } return page, nil } options.Cursor = history.Next } + if selecting { + // A bounded scan cannot prove absence or uniqueness beyond its horizon. + page.SelectedOccurrence, page.SelectionStatus = nil, "partial" + page.Sync.Partial = true + page.Partial = append(page.Partial, PartialFailure{Class: "selection_scan_limit"}) + return page, nil + } return page, storage.ErrEvidenceLimit } diff --git a/pkg/kubernetes/evidence_review_regression_test.go b/pkg/kubernetes/evidence_review_regression_test.go index 52d423fb..a2a3ddb8 100644 --- a/pkg/kubernetes/evidence_review_regression_test.go +++ b/pkg/kubernetes/evidence_review_regression_test.go @@ -366,6 +366,117 @@ func TestPersistedRootContextIgnoresEvidenceWindowAndLiveExistence(t *testing.T) } } +func TestPersistedRootSelectionNeverSubstitutesAnotherOccurrence(t *testing.T) { + for _, backend := range []string{"memory", "file"} { + t.Run(backend, func(t *testing.T) { + service, store, scope := reviewHistoryService(t) + if backend == "file" { + provider, err := storage.NewFile(storage.FileOptions{DataDir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { provider.Close() }) + service.SetChangeStorage(provider) + store = provider.(storage.EvidenceStore) + } + at := time.Now().UTC().Add(-2 * time.Hour).Truncate(time.Second).Add(123456789 * time.Nanosecond) + root := ObjectRef{Kind: "Deployment", Namespace: "shop", Name: "retained", UID: "retained-root"} + batch := storage.EvidenceBatch{ID: "capture:selected", At: at, Epoch: "synthetic"} + for i := 0; i < 3; i++ { + broke := at + if i == 2 { + broke = at.Add(time.Minute) + } + rule := fmt.Sprintf("synthetic.rule-%d", i) + raw, _ := json.Marshal(Issue{Root: root, Rule: rule, Broken: true, BrokeAt: &broke, Severity: "warning"}) + batch.Issues = append(batch.Issues, storage.EvidenceIssue{Key: rule, Namespace: root.Namespace, Kind: root.Kind, UID: root.UID, Rule: rule, LastSeen: at, ObservationOnly: true, Payload: raw}) + } + if err := store.CommitEvidence(t.Context(), scope, batch); err != nil { + t.Fatal(err) + } + history, err := service.historicalIssues(t.Context(), IssueOptions{RootUID: root.UID, Namespace: root.Namespace, Since: at.Add(-time.Minute), Until: time.Now()}) + if err != nil || len(history.Items) != 3 { + t.Fatal("synthetic retained occurrences unavailable") + } + selected := history.Items[2] + for _, test := range []struct { + name, id, namespace, uid, status string + center time.Time + }{ + {name: "exact", id: selected.OccurrenceID, status: "resolved"}, + {name: "mismatch", id: "missing-id", center: *selected.BrokeAt, status: "not_found"}, + {name: "unique-center", center: at.Add(time.Minute), status: "resolved"}, + {name: "millisecond-center-does-not-substitute", center: at.Add(time.Minute).Truncate(time.Millisecond), status: "not_found"}, + {name: "nearby-center-does-not-substitute", center: at.Add(time.Minute + time.Nanosecond), status: "not_found"}, + {name: "ambiguous-center", center: at, status: "ambiguous"}, + {name: "no-selector", status: "not_requested"}, + {name: "wrong-namespace", id: selected.OccurrenceID, namespace: "other", status: "not_found"}, + {name: "same-name-new-uid", id: selected.OccurrenceID, uid: "replacement-root", status: "not_found"}, + } { + t.Run(test.name, func(t *testing.T) { + options := IssueOptions{Context: "persisted", RootUID: root.UID, Namespace: root.Namespace} + if test.namespace != "" { + options.Namespace = test.namespace + } + if test.uid != "" { + options.RootUID = test.uid + } + options.OccurrenceID, options.Center = test.id, test.center + page, err := service.Issues(t.Context(), options) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(page) + var response struct { + Status string `json:"selection_status"` + Selected *Issue `json:"selected_occurrence"` + } + if err := json.Unmarshal(raw, &response); err != nil { + t.Fatal(err) + } + if response.Status != test.status { + t.Fatalf("selection status=%q want=%q", response.Status, test.status) + } + if test.status != "resolved" && response.Selected != nil { + t.Fatal("unresolved selector substituted another recurrence") + } + if test.name == "exact" && (response.Selected == nil || response.Selected.OccurrenceID != selected.OccurrenceID) { + t.Fatal("explicit occurrence did not survive date-free root lookup") + } + if test.name == "unique-center" && (response.Selected == nil || response.Selected.BrokeAt == nil || !response.Selected.BrokeAt.Equal(test.center)) { + t.Fatal("nanosecond center did not preserve the exact retained occurrence") + } + if test.name == "no-selector" && (page.RootContext == nil || response.Selected != nil) { + t.Fatal("legacy root link lost retained context or selected an arbitrary occurrence") + } + if len(page.Items) != 0 || len(page.Totals) != 0 || page.Next != "" { + t.Fatal("selected context polluted historical paging") + } + }) + } + }) + } +} + +func TestPersistedRootSelectionReportsBoundedScanWithoutFalseUniqueness(t *testing.T) { + service, store, scope := reviewHistoryService(t) + at := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + root := ObjectRef{Kind: "Deployment", Namespace: "shop", Name: "retained", UID: "retained-root"} + batch := storage.EvidenceBatch{ID: "capture:many", At: at, Epoch: "synthetic"} + for i := 0; i < 1001; i++ { + rule := fmt.Sprintf("synthetic.rule-%04d", i) + raw, _ := json.Marshal(Issue{Root: root, Rule: rule, BrokeAt: &at, Broken: true, Severity: "warning"}) + batch.Issues = append(batch.Issues, storage.EvidenceIssue{Key: rule, Namespace: root.Namespace, Kind: root.Kind, UID: root.UID, Rule: rule, LastSeen: at, ObservationOnly: true, Payload: raw}) + } + if err := store.CommitEvidence(t.Context(), scope, batch); err != nil { + t.Fatal(err) + } + page, err := service.Issues(t.Context(), IssueOptions{Context: "persisted", RootUID: root.UID, Namespace: root.Namespace, OccurrenceID: "not-in-scanned-prefix"}) + if err != nil || page.SelectionStatus != "partial" || page.SelectedOccurrence != nil || page.RootContext == nil || !page.Sync.Partial { + t.Fatal("bounded scan claimed absence or erased retained root") + } +} + func TestScopedEvidenceProjectsUnavailableRecordsBeforeCountsAndPagination(t *testing.T) { for _, backend := range []string{"memory", "file", "postgres"} { t.Run(backend, func(t *testing.T) { @@ -502,6 +613,11 @@ func TestScopedEvidenceProjectsUnavailableRecordsBeforeCountsAndPagination(t *te if err != nil || contextPage.RootContext == nil || contextPage.RootContext.Root.UID != "retained-root" || len(contextPage.Items) != 0 || !contextPage.Sync.Partial { t.Fatal("persisted context required a live object, hid a malformed observation, or leaked into history") } + contextOptions.OccurrenceID = contextPage.RootContext.Occurrence.OccurrenceID + selectedPage, err := service.Issues(t.Context(), contextOptions) + if err != nil || selectedPage.SelectedOccurrence == nil || selectedPage.SelectionStatus != "resolved" || selectedPage.SelectedOccurrence.BrokeAt != nil { + t.Fatal("exact missing-time occurrence was substituted or fabricated a timestamp") + } occurrences, err := service.queryEvidence(t.Context(), storage.EvidenceQuery{Since: at.Add(-time.Minute), Until: at.Add(time.Minute), Namespace: "sample", RootUID: "retained-root", Type: "occurrence", Limit: 1}) if err != nil || occurrences.Total != 2 || occurrences.Counts["issue_opened"] != 1 || occurrences.Counts["coverage_gap"] != 1 || len(occurrences.Items) != 1 || occurrences.Next == "" { t.Fatal("occurrence projection did not preserve accurate counts and bounded opaque pagination") @@ -580,6 +696,9 @@ func TestScopedEvidenceProjectsUnavailableRecordsBeforeCountsAndPagination(t *te } denyPod.Store(false) wrongCluster.Store(true) + if selected, err := service.Issues(t.Context(), contextOptions); err != nil || selected.SelectedOccurrence != nil || selected.RootContext != nil || selected.SelectionStatus != "not_found" { + t.Fatal("selected occurrence bypassed physical cluster identity") + } page, err := service.queryEvidence(t.Context(), global) if err != nil || len(page.Items) != 0 { t.Fatal("physical cluster replacement exposed retained evidence") diff --git a/pkg/kubernetes/inventory_cache.go b/pkg/kubernetes/inventory_cache.go index 3e29ffde..4f00be10 100644 --- a/pkg/kubernetes/inventory_cache.go +++ b/pkg/kubernetes/inventory_cache.go @@ -278,17 +278,24 @@ func (service *Service) CachedNamespaceSnapshot(ctx context.Context, namespace s } if scoped, exists := generation.Namespaces[namespace]; namespace != "" && exists { states := make(map[string]kubeindex.KindStatus, len(generation.View.Kinds)) + scopedKinds := make(map[string]bool, len(scoped.Kinds)) for kind, state := range generation.View.Kinds { states[kind] = state } for kind, state := range scoped.Kinds { - if states[kind].State != "ready" { + global := states[kind] + // Namespace authority can replace a global permission limitation, + // not a newer transient observation or a whole-cycle auth failure. + // A newer scoped failure is authoritative too; never hide a denial. + authFailure := global.CycleFailure && (global.Error == "forbidden" || global.Error == "unauthorized") + if !authFailure && (global.Error == "forbidden" || global.Error == "namespace_scope" || inventoryObservationAt(state).After(inventoryObservationAt(global))) { states[kind] = state + scopedKinds[kind] = true } } records := make([]kubeindex.Record, 0) for _, record := range generation.View.Records { - if record.Namespace == "" || record.Namespace == namespace { + if record.Namespace == "" || record.Namespace == namespace && !scopedKinds[record.Kind] { records = append(records, record) } } @@ -297,6 +304,12 @@ func (service *Service) CachedNamespaceSnapshot(ctx context.Context, namespace s byUID[record.UID] = true } for _, record := range scoped.Records { + if !scopedKinds[record.Kind] || record.Namespace != namespace { + // Choose one committed source per kind, never their union. A + // complete replacement (global or scoped) includes real deletions; + // records from an older fallback must not resurrect removed rows. + continue + } if !byUID[record.UID] { records = append(records, record) } @@ -386,6 +399,19 @@ func (service *Service) CachedNamespaceSnapshot(ctx context.Context, namespace s return generation.View, status, nil } +// inventoryObservationAt orders authority observations, including failed LISTs +// that retain the last successful ObservedAt and its committed records. +func inventoryObservationAt(state kubeindex.KindStatus) time.Time { + at := state.ObservedAt + if state.LastAttemptAt.After(at) { + at = state.LastAttemptAt + } + if state.FailingSince.After(at) { + at = state.FailingSince + } + return at +} + // pendingKinds counts the resource types a published generation still lacks. func (generation inventoryGeneration) pendingKinds() int { if len(generation.View.Kinds) == 0 { diff --git a/pkg/kubernetes/inventory_cache_test.go b/pkg/kubernetes/inventory_cache_test.go index a29329cf..d455ec54 100644 --- a/pkg/kubernetes/inventory_cache_test.go +++ b/pkg/kubernetes/inventory_cache_test.go @@ -31,6 +31,244 @@ func seedInventory(t *testing.T, service *Service, records []kubeindex.Record) { } } +func TestWorkloadsPinsPartialPublicationAcrossReplacement(t *testing.T) { + client, err := NewClient(Config{Endpoint: "http://127.0.0.1:1", AllowLoopbackHTTP: true, Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + service := NewService(client, Scope{OrgID: "default", ClusterID: "cluster"}, 0) + old := []kubeindex.Record{ + {UID: "old-pod", Kind: "Pod", Namespace: "shop", Name: "old-pod"}, + {UID: "old-deployment", Kind: "Deployment", Namespace: "shop", Name: "old-deployment"}, + } + seedInventory(t, service, old) + cache := service.changes.inventories[service.indexScope()] + state := cache.generation.View.Kinds["Pod"] + state.State, state.Partial, state.Error = "error", true, "timeout" + cache.generation.View.Kinds["Pod"] = state + var reads atomic.Int32 + client.credentials = &credentialSource{authorization: func(context.Context) (string, error) { + if reads.Add(1) == 2 { + cache.mu.Lock() + replacement, cloneErr := cloneInventoryGeneration(cache.generation) + if cloneErr != nil { + t.Fatal(cloneErr) + } + replacement.View.Records = []kubeindex.Record{{UID: "new-pod", Kind: "Pod", Namespace: "shop", Name: "new-pod"}} + cache.generation = replacement + cache.mu.Unlock() + } + return "", nil + }} + page, err := service.Workloads(t.Context(), WorkloadListOptions{Namespace: "shop"}) + if err != nil || len(page.Items) != 2 || page.Counts["Deployment"] != 1 || page.Counts["Pod"] != 1 { + t.Fatalf("partial committed publication lost rows across replacement: rows=%d err=%v", len(page.Items), err) + } + for _, item := range page.Items { + if item.UID != "old-pod" && item.UID != "old-deployment" { + t.Fatal("one response mixed inventory generations") + } + } + if len(page.Partial) != 1 || page.Partial[0].Class != "timeout" { + t.Fatal("retained rows concealed the transient collection failure") + } +} + +func TestNamespaceProjectionDoesNotResurrectDeletedGlobalRecords(t *testing.T) { + service := NewService(nil, Scope{OrgID: "default", ClusterID: "cluster"}, 0) + seedInventory(t, service, nil) + cache := service.changes.inventories[service.indexScope()] + cache.generation.Namespaces = map[string]kubeindex.Snapshot{"shop": { + Records: []kubeindex.Record{{UID: "deleted-pod", Kind: "Pod", Namespace: "shop", Name: "reused-name"}}, + Kinds: map[string]kubeindex.KindStatus{"Pod": { + State: "ready", ObservedAt: time.Now().Add(-time.Minute), + }}, + }} + for _, namespace := range []string{"", "shop", "other"} { + page, err := service.Workloads(t.Context(), WorkloadListOptions{Namespace: namespace}) + if err != nil || len(page.Items) != 0 || page.Counts["Pod"] != 0 { + t.Fatalf("complete empty replacement resurrected scoped rows: namespace=%q rows=%d err=%v", namespace, len(page.Items), err) + } + } +} + +func TestNamespaceProjectionPrefersNewerCompleteScopedReplacement(t *testing.T) { + service := NewService(nil, Scope{OrgID: "default", ClusterID: "cluster"}, 0) + seedInventory(t, service, []kubeindex.Record{{UID: "deleted-pod", Kind: "Pod", Namespace: "shop", Name: "reused-name"}}) + cache := service.changes.inventories[service.indexScope()] + state := cache.generation.View.Kinds["Pod"] + state.ObservedAt = time.Now().Add(-time.Minute) + cache.generation.View.Kinds["Pod"] = state + cache.generation.Namespaces = map[string]kubeindex.Snapshot{"shop": { + Kinds: map[string]kubeindex.KindStatus{"Pod": { + State: "ready", ObservedAt: time.Now(), + }}, + }} + page, err := service.Workloads(t.Context(), WorkloadListOptions{Namespace: "shop"}) + if err != nil || len(page.Items) != 0 || page.Counts["Pod"] != 0 { + t.Fatal("newer scoped empty replacement resurrected an older global row") + } +} + +func TestNamespaceProjectionPreservesObservationAuthorityAndFailure(t *testing.T) { + for _, test := range []struct { + name, globalError, scopedError, wantUID, wantError string + newer, cycle bool + }{ + {name: "stale-scoped-timeout", globalError: "timeout", wantUID: "current-pod", wantError: "timeout"}, + {name: "stale-scoped-unauthorized", globalError: "unauthorized", wantError: "unauthorized"}, + {name: "global-forbidden", globalError: "forbidden", wantUID: "scoped-pod"}, + {name: "global-namespace-scope", globalError: "namespace_scope", wantUID: "scoped-pod"}, + {name: "newer-scoped-ready", globalError: "timeout", newer: true, wantUID: "scoped-pod"}, + {name: "newer-scoped-timeout", scopedError: "timeout", newer: true, wantUID: "scoped-pod", wantError: "timeout"}, + {name: "newer-scoped-denial", scopedError: "forbidden", newer: true, wantError: "forbidden"}, + {name: "whole-cycle-denial", globalError: "forbidden", cycle: true}, + } { + t.Run(test.name, func(t *testing.T) { + service := NewService(nil, Scope{OrgID: "default", ClusterID: "cluster"}, 0) + seedInventory(t, service, []kubeindex.Record{{UID: "current-pod", Kind: "Pod", Namespace: "shop", Name: "reused-name"}}) + cache := service.changes.inventories[service.indexScope()] + now := time.Now().UTC() + // A real failed refresh advances the attempt, not the successful LIST. + global := kubeindex.KindStatus{State: "ready", ObservedAt: now.Add(-12 * time.Hour), LastAttemptAt: now} + if test.globalError != "" { + global.State, global.Partial, global.Error, global.CycleFailure = "error", true, test.globalError, test.cycle + } + cache.generation.View.Kinds["Pod"] = global + scoped := kubeindex.KindStatus{State: "ready", ObservedAt: now.Add(-6 * time.Hour)} + scoped.LastAttemptAt = scoped.ObservedAt + if test.newer { + scoped.ObservedAt = now.Add(time.Second) + scoped.LastAttemptAt = scoped.ObservedAt + } + if test.scopedError != "" { + scoped.State, scoped.Partial, scoped.Error = "error", true, test.scopedError + } + cache.generation.Namespaces = map[string]kubeindex.Snapshot{"shop": { + Records: []kubeindex.Record{{UID: "scoped-pod", Kind: "Pod", Namespace: "shop", Name: "reused-name"}}, + Kinds: map[string]kubeindex.KindStatus{"Pod": scoped}, + }} + page, err := service.Workloads(t.Context(), WorkloadListOptions{Namespace: "shop", Kind: "Pod"}) + if test.cycle { + if !errors.Is(err, ErrForbidden) { + t.Fatalf("whole-cycle denial concealed by scoped cache: %v", err) + } + return + } + if err != nil { + t.Fatal(err) + } + if test.wantUID == "" { + if len(page.Items) != 0 { + t.Fatal("denied observation exposed records") + } + } else if len(page.Items) != 1 || page.Items[0].UID != test.wantUID { + t.Fatalf("wrong committed source: %+v", page.Items) + } + if test.wantError == "" { + if len(page.Partial) != 0 { + t.Fatalf("scoped authority did not clear global limitation: %+v", page.Partial) + } + } else if len(page.Partial) != 1 || page.Partial[0].Class != test.wantError { + t.Fatalf("observation failure concealed: %+v", page.Partial) + } + }) + } +} + +func TestWorkloadCountsBeforePublicationAndCommittedEmpty(t *testing.T) { + service := NewService(nil, Scope{OrgID: "default", ClusterID: "cluster"}, 0) + for _, namespace := range []string{"", "shop"} { + page, err := service.Workloads(t.Context(), WorkloadListOptions{Namespace: namespace}) + if err != nil || len(page.Items) != 0 || len(page.Counts) != 0 || page.Sync.State != "syncing" || page.Sync.Partial || len(page.Sync.Pending) != len(workloadResources) || page.Sync.PendingKinds != len(indexedResourceIDs) { + t.Fatalf("no publication manufactured known empty counts: namespace=%q page=%+v err=%v", namespace, page, err) + } + } + seedInventory(t, service, nil) + for _, namespace := range []string{"", "shop"} { + page, err := service.Workloads(t.Context(), WorkloadListOptions{Namespace: namespace}) + if err != nil || len(page.Items) != 0 || len(page.Counts) != len(workloadResources) || len(page.Sync.Pending) != 0 { + t.Fatalf("committed empty listing lost known counts: namespace=%q page=%+v err=%v", namespace, page, err) + } + for kind, count := range page.Counts { + if count != 0 { + t.Fatalf("committed empty %s count=%d", kind, count) + } + } + } +} + +func TestWorkloadCursorBindsCommittedSnapshotAndFilters(t *testing.T) { + service := NewService(nil, Scope{OrgID: "default", ClusterID: "cluster"}, 0) + records := []kubeindex.Record{ + {UID: "c", Kind: "Deployment", Namespace: "shop", Name: "web"}, + {UID: "a", Kind: "Deployment", Namespace: "payments", Name: "ledger"}, + {UID: "b", Kind: "Deployment", Namespace: "shop", Name: "only-a"}, + } + seedInventory(t, service, records) + first, err := service.Workloads(t.Context(), WorkloadListOptions{Kind: "Deployment", Limit: 1}) + if err != nil || len(first.Items) != 1 || first.Items[0].UID != "a" || first.Next == "" { + t.Fatalf("first sorted page: %+v %v", first, err) + } + second, err := service.Workloads(t.Context(), WorkloadListOptions{Kind: "Deployment", Limit: 1, Cursor: first.Next}) + if err != nil || len(second.Items) != 1 || second.Items[0].UID != "b" { + t.Fatalf("valid continuation: %+v %v", second, err) + } + for _, options := range []WorkloadListOptions{ + {Kind: "Pod", Cursor: first.Next}, + {Kind: "Deployment", Namespace: "shop", Cursor: first.Next}, + {Kind: "Deployment", Query: "web", Cursor: first.Next}, + {Kind: "Deployment", Cursor: "1"}, + {Kind: "Deployment", Cursor: first.Next + "corrupt"}, + } { + if _, err := service.Workloads(t.Context(), options); !errors.Is(err, ErrInvalidArguments) { + t.Fatalf("unbound cursor accepted: %+v %v", options, err) + } + } + scoped, err := service.Workloads(t.Context(), WorkloadListOptions{Namespace: "shop", Kind: "Deployment", Limit: 1}) + if err != nil || len(scoped.Items) != 1 || scoped.Items[0].UID != "b" || scoped.Next == "" { + t.Fatalf("scoped sorting: %+v %v", scoped, err) + } + scopedNext, err := service.Workloads(t.Context(), WorkloadListOptions{Namespace: "shop", Kind: "Deployment", Limit: 1, Cursor: scoped.Next}) + if err != nil || len(scopedNext.Items) != 1 || scopedNext.Items[0].UID != "c" || scopedNext.Next != "" { + t.Fatalf("scoped continuation: %+v %v", scopedNext, err) + } + cache := service.changes.inventories[service.indexScope()] + for kind, state := range cache.generation.View.Kinds { + state.ObservedAt, state.CompletedAt, state.LastAttemptAt = time.Now(), time.Now(), time.Now() + cache.generation.View.Kinds[kind] = state + } + cache.generation.View.Kinds["Event"] = kubeindex.KindStatus{State: "error", Error: "timeout", ObservedAt: time.Now()} + unchanged, err := service.Workloads(t.Context(), WorkloadListOptions{Kind: "Deployment", Limit: 1, Cursor: first.Next}) + if err != nil || len(unchanged.Items) != 1 || unchanged.Items[0].UID != "b" { + t.Fatalf("timestamps or unrelated kinds expired unchanged page: %+v %v", unchanged, err) + } + cache.generation.View.Records = append(records, kubeindex.Record{UID: "inserted", Kind: "Deployment", Namespace: "payments", Name: "aaa"}) + if _, err := service.Workloads(t.Context(), WorkloadListOptions{Kind: "Deployment", Cursor: first.Next}); !errors.Is(err, ErrInvalidArguments) { + t.Fatalf("replacement cursor did not return typed invalidation: %v", err) + } +} + +func TestWorkloadPendingKindCountIsUnknown(t *testing.T) { + service := NewService(nil, Scope{OrgID: "default", ClusterID: "cluster"}, 0) + seedInventory(t, service, []kubeindex.Record{{UID: "pod", Kind: "Pod", Namespace: "shop", Name: "pod"}}) + cache := service.changes.inventories[service.indexScope()] + delete(cache.generation.View.Kinds, "CronJob") + cache.generation.Pending = []string{indexedResourceIDs["CronJob"]} + for _, namespace := range []string{"", "shop"} { + page, err := service.Workloads(t.Context(), WorkloadListOptions{Namespace: namespace}) + if err != nil { + t.Fatal(err) + } + if _, known := page.Counts["CronJob"]; known { + t.Fatalf("pending CronJob is a known zero: %+v", page.Counts) + } + if page.Counts["Pod"] != 1 || page.Counts["Deployment"] != 0 || !containsString(page.Sync.Pending, indexedResourceIDs["CronJob"]) || len(page.Partial) != 0 || page.Truncated { + t.Fatalf("pending changed successful counts or became failure: %+v", page) + } + } +} + func TestCachedNamespaceSnapshotIgnoresRecordCounts(t *testing.T) { maxInt := int(^uint(0) >> 1) for _, test := range []struct { @@ -885,6 +1123,16 @@ func TestPendingKindsNeverAppearAsFailuresInAnyView(t *testing.T) { } } sync, _ := body["sync"].(map[string]any) + if name == "workloads" { + counts, _ := body["counts"].(map[string]any) + for _, candidate := range workloadResources { + if pendingIDs[candidate.ResourceID] { + if _, known := counts[candidate.Kind]; known { + t.Errorf("pending %s has a known count: %v", candidate.Kind, counts) + } + } + } + } if sync["state"] != "unavailable" && sync["pending_kinds"] == nil { t.Errorf("%s does not say other kinds are loading: %v", name, sync) } diff --git a/pkg/kubernetes/inventory_continuity_adversarial_test.go b/pkg/kubernetes/inventory_continuity_adversarial_test.go new file mode 100644 index 00000000..c7dc2243 --- /dev/null +++ b/pkg/kubernetes/inventory_continuity_adversarial_test.go @@ -0,0 +1,142 @@ +package kubernetes + +import ( + "errors" + "testing" + "time" + + kubeindex "github.com/VersusControl/versus-incident/pkg/kubernetes/index" +) + +func TestCommittedReplacementNeverRebindsSameNameUID(t *testing.T) { + for _, newerScoped := range []bool{false, true} { + t.Run(map[bool]string{false: "global", true: "namespace"}[newerScoped], func(t *testing.T) { + service := NewService(nil, Scope{OrgID: "default", ClusterID: "cluster"}, 0) + old := kubeindex.Record{UID: "retired-root", Kind: "StatefulSet", Namespace: "shop", Name: "api"} + replacement := old + replacement.UID = "replacement-root" + child := kubeindex.Record{UID: "replacement-pod", Kind: "Pod", Namespace: "shop", Name: "api-0", + Owners: []kubeindex.OwnerRef{{UID: replacement.UID, Kind: replacement.Kind, Name: replacement.Name, Namespace: "shop", ControllerKnown: true, Controller: true}}} + seedInventory(t, service, []kubeindex.Record{replacement, child}) + cache := service.changes.inventories[service.indexScope()] + now := time.Now().UTC() + states := map[string]kubeindex.KindStatus{ + "StatefulSet": {State: "ready", ObservedAt: now.Add(-time.Minute)}, + "Pod": {State: "ready", ObservedAt: now.Add(-time.Minute)}, + } + cache.generation.Namespaces = map[string]kubeindex.Snapshot{"shop": {Records: []kubeindex.Record{old}, Kinds: states}} + if newerScoped { + cache.generation.View.Records = []kubeindex.Record{old} + cache.generation.Namespaces["shop"] = kubeindex.Snapshot{Records: []kubeindex.Record{replacement, child}, + Kinds: map[string]kubeindex.KindStatus{"StatefulSet": {State: "ready", ObservedAt: now.Add(time.Second)}, "Pod": {State: "ready", ObservedAt: now.Add(time.Second)}}} + } + snapshot, _, err := service.CachedNamespaceSnapshot(t.Context(), "shop", "StatefulSet", "Pod") + if err != nil { + t.Fatal(err) + } + options := WorkloadLogStreamOptions{Kind: "StatefulSet", Namespace: "shop", Name: "api", UID: old.UID} + if pods, err := workloadLogPods(snapshot, options, true); !errors.Is(err, ErrNotFound) || len(pods) != 0 { + t.Fatalf("retired UID was rebound to same-name replacement: pods=%+v err=%v", pods, err) + } + options.UID = replacement.UID + pods, err := workloadLogPods(snapshot, options, true) + if err != nil || len(pods) != 1 || pods[0].UID != child.UID { + t.Fatalf("replacement membership=%+v err=%v", pods, err) + } + options.Namespace = "other" + if pods, err := workloadLogPods(snapshot, options, true); !errors.Is(err, ErrNotFound) || len(pods) != 0 { + t.Fatalf("membership leaked namespace: pods=%+v err=%v", pods, err) + } + }) + } +} + +func TestWorkloadsOptionalKindFailurePreservesScopedCountsAndEmptyReplacement(t *testing.T) { + for _, failure := range []string{"pending", "timeout", "forbidden"} { + t.Run(failure, func(t *testing.T) { + service := NewService(nil, Scope{OrgID: "default", ClusterID: "cluster"}, 0) + seedInventory(t, service, []kubeindex.Record{ + {UID: "shop-deploy", Kind: "Deployment", Namespace: "shop", Name: "api"}, + {UID: "other-deploy", Kind: "Deployment", Namespace: "other", Name: "api"}, + {UID: "shop-cron", Kind: "CronJob", Namespace: "shop", Name: "cleanup"}, + }) + cache := service.changes.inventories[service.indexScope()] + state := cache.generation.View.Kinds["CronJob"] + if failure == "pending" { + state = kubeindex.KindStatus{State: "pending"} + cache.generation.Pending = []string{indexedResourceIDs["CronJob"]} + // An uncommitted optional kind cannot supply rows. + cache.generation.View.Records = cache.generation.View.Records[:2] + } else { + state.State, state.Partial, state.Error = "error", true, failure + } + cache.generation.View.Kinds["CronJob"] = state + page, err := service.Workloads(t.Context(), WorkloadListOptions{Namespace: "shop"}) + wantCron := 0 + if failure == "timeout" { + wantCron = 1 + } + if err != nil || len(page.Items) != 1+wantCron || page.Counts["Deployment"] != 1 || page.Counts["CronJob"] != wantCron { + t.Fatalf("optional %s erased or leaked committed projection: page=%+v err=%v", failure, page, err) + } + if _, known := page.Counts["CronJob"]; known != (failure == "timeout") { + t.Fatalf("optional %s count provenance: counts=%v", failure, page.Counts) + } + if failure != "pending" && (len(page.Partial) != 1 || page.Partial[0].Class != failure) { + t.Fatalf("optional failure concealed: %+v", page.Partial) + } + // A complete empty publication must erase the old scoped inventory, + // including after a retained partial publication. + seedInventory(t, service, nil) + for _, namespace := range []string{"shop", "other", ""} { + empty, err := service.Workloads(t.Context(), WorkloadListOptions{Namespace: namespace}) + if err != nil || len(empty.Items) != 0 || empty.Counts["Deployment"] != 0 || empty.Counts["CronJob"] != 0 { + t.Fatalf("empty replacement resurrected rows: namespace=%q page=%+v err=%v", namespace, empty, err) + } + for _, candidate := range workloadResources { + if count, known := empty.Counts[candidate.Kind]; !known || count != 0 { + t.Fatalf("successfully committed empty %s count=%d known=%t", candidate.Kind, count, known) + } + } + } + }) + } +} + +func TestWorkloadsCountRequiresSuccessfulAuthorizedObservation(t *testing.T) { + for _, failure := range []string{"timeout", "unavailable", "throttled", "forbidden", "unauthorized", "namespace_scope", "credential_unavailable"} { + for _, observed := range []bool{false, true} { + t.Run(failure+"/"+map[bool]string{false: "never-observed", true: "last-good"}[observed], func(t *testing.T) { + service := NewService(nil, Scope{OrgID: "default", ClusterID: "cluster"}, 0) + records := []kubeindex.Record{{UID: "shop-deploy", Kind: "Deployment", Namespace: "shop", Name: "api"}} + if observed { + records = append(records, kubeindex.Record{UID: "shop-cron", Kind: "CronJob", Namespace: "shop", Name: "cleanup"}) + } + seedInventory(t, service, records) + cache := service.changes.inventories[service.indexScope()] + state := cache.generation.View.Kinds["CronJob"] + state.State, state.Error, state.Partial = "error", failure, true + state.LastAttemptAt = time.Now().UTC() + if !observed { + state.ObservedAt, state.CompletedAt = time.Time{}, time.Time{} + } + cache.generation.View.Kinds["CronJob"] = state + page, err := service.Workloads(t.Context(), WorkloadListOptions{Namespace: "shop"}) + if err != nil { + t.Fatal(err) + } + wantKnown := observed && (failure == "timeout" || failure == "unavailable" || failure == "throttled") + count, known := page.Counts["CronJob"] + if known != wantKnown || wantKnown && count != 1 { + t.Fatalf("count=%d known=%t wantKnown=%t page=%+v", count, known, wantKnown, page) + } + if len(page.Partial) != 1 || page.Partial[0].Class != failure { + t.Fatalf("failure lost: %+v", page) + } + if containsString(page.Sync.Pending, indexedResourceIDs["CronJob"]) || page.Sync.State == "ready" { + t.Fatalf("failed LIST changed into pending or ready: %+v", page.Sync) + } + }) + } + } +} diff --git a/pkg/kubernetes/operations.go b/pkg/kubernetes/operations.go index 84e69cb1..298e909b 100644 --- a/pkg/kubernetes/operations.go +++ b/pkg/kubernetes/operations.go @@ -2,6 +2,8 @@ package kubernetes import ( "context" + "crypto/sha256" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -10,6 +12,8 @@ import ( "strings" "sync" "time" + + kubeindex "github.com/VersusControl/versus-incident/pkg/kubernetes/index" ) const ( @@ -702,36 +706,39 @@ func (service *Service) Workloads(ctx context.Context, options WorkloadListOptio return WorkloadPage{}, indexErr } result.Sync = SyncStatus{State: status.State, Partial: status.Partial, AgeSeconds: status.AgeSeconds, Progress: status.Progress, Pending: status.Pending, PendingKinds: status.PendingKinds} - indexReady := true for _, candidate := range workloadResources { - if status.Kinds[candidate.Kind].State != "ready" { - indexReady = false - break - } - } - if indexReady { - for _, record := range view.Records { - if !isWorkloadKind(record.Kind) { - continue + state, known := status.Kinds[candidate.Kind] + if !known || state.State == "pending" || containsString(status.Pending, candidate.ResourceID) { + // Missing means unknown, not a successful count of zero. Keep the + // pending resource IDs in sync without manufacturing a failure. + delete(result.Counts, candidate.Kind) + if !containsString(result.Sync.Pending, candidate.ResourceID) { + result.Sync.Pending = append(result.Sync.Pending, candidate.ResourceID) } - resources = append(resources, projectedWorkloadIndexRecord(record)) } + // Only a successful, still-authorized observation establishes a count. + // Transient failures retain that observation and its committed rows; + // a first LIST failure is an explicit error, not pending or known empty. + if state.ObservedAt.IsZero() || state.Error == "forbidden" || state.Error == "unauthorized" || + state.Error == "namespace_scope" || state.Error == "credential_unavailable" { + delete(result.Counts, candidate.Kind) + } + if state.Error != "" { + result.Partial = append(result.Partial, PartialFailure{ResourceID: candidate.ResourceID, Class: state.Error}) + } + } + if len(status.Kinds) == 0 && status.State == "syncing" { + result.Sync.PendingKinds = len(indexedResourceIDs) } else { - for _, candidate := range workloadResources { - page, listErr := service.cachedList(ctx, ListOptions{ResourceID: candidate.ResourceID, Namespace: options.Namespace}) - if listErr != nil { - class := errorClass(listErr) - if errors.Is(listErr, ErrNotFound) { - class = "unsupported" - } - result.Partial = append(result.Partial, PartialFailure{ResourceID: candidate.ResourceID, Class: class}) - result.Truncated = true - continue - } - resources = append(resources, page.Items...) - result.Partial = append(result.Partial, page.Partial...) - result.Truncated = result.Truncated || page.Truncated - result.Omitted = append(result.Omitted, page.Omitted...) + result.Sync.PendingKinds = max(result.Sync.PendingKinds, len(result.Sync.Pending)) + } + // Snapshot projection already excludes denied kinds and retains committed + // records through transient refresh failures. Reading each kind again here + // could cross a publication boundary and mix rows, counts and observation + // times from different generations. + for _, record := range view.Records { + if isWorkloadKind(record.Kind) { + resources = append(resources, projectedWorkloadIndexRecord(record)) } } query := strings.ToLower(strings.TrimSpace(options.Query)) @@ -740,7 +747,9 @@ func (service *Service) Workloads(ctx context.Context, options WorkloadListOptio if options.Namespace != "" && resource.Namespace != options.Namespace || query != "" && !strings.Contains(strings.ToLower(resource.Name), query) && !strings.Contains(strings.ToLower(resource.Namespace), query) { continue } - result.Counts[resource.Kind]++ + if _, known := result.Counts[resource.Kind]; known { + result.Counts[resource.Kind]++ + } if options.Kind == "" || resource.Kind == options.Kind { filtered = append(filtered, resource) } @@ -754,22 +763,74 @@ func (service *Service) Workloads(ctx context.Context, options WorkloadListOptio } return filtered[i].Name < filtered[j].Name }) - start, end, next, err := pageWindow(options.Cursor, limit, len(filtered)) + // Bind the offset to the committed projection and normalized filters read + // above. A replacement cannot turn an old offset into duplicate-row success. + // Observation timestamps and unrelated kinds do not change the page, so + // routine polling alone must not expire a valid continuation. + type cursorKindState struct { + State, Error string + Partial, CycleFailure bool + } + cursorKinds := make(map[string]cursorKindState, len(workloadResources)) + for _, candidate := range workloadResources { + state := status.Kinds[candidate.Kind] + cursorKinds[candidate.Kind] = cursorKindState{state.State, state.Error, state.Partial, state.CycleFailure} + } + binding, err := json.Marshal(struct { + Scope kubeindex.Scope + Namespace, Kind, Query string + Rows []ProjectedResource + Counts map[string]int + Kinds map[string]cursorKindState + }{service.indexScope(), options.Namespace, options.Kind, query, filtered, result.Counts, cursorKinds}) + if err != nil { + return WorkloadPage{}, err + } + digest := fmt.Sprintf("%x", sha256.Sum256(binding)) + offset, err := workloadCursorOffset(options.Cursor, digest) + if err != nil { + return WorkloadPage{}, err + } + start, end, next, err := pageWindow(offset, limit, len(filtered)) if err != nil { return WorkloadPage{}, err } result.Items = append(result.Items, filtered[start:end]...) - result.Next = next + result.Next = encodeWorkloadCursor(next, digest) var sizeTruncated bool result.Items, result.Omitted, sizeTruncated = trimAggregateItems(result.Items, result.Omitted) result.Truncated = result.Truncated || sizeTruncated if sizeTruncated { - start, _ := strconv.Atoi(options.Cursor) - result.Next = strconv.Itoa(start + len(result.Items)) + result.Next = encodeWorkloadCursor(strconv.Itoa(start+len(result.Items)), digest) } return result, nil } +func workloadCursorOffset(cursor, digest string) (string, error) { + if cursor == "" { + return "", nil + } + if len(cursor) > 128 { + return "", ErrInvalidArguments + } + raw, err := base64.RawURLEncoding.DecodeString(cursor) + if err != nil { + return "", ErrInvalidArguments + } + binding, offset, ok := strings.Cut(string(raw), ":") + if !ok || binding != digest || offset == "" { + return "", ErrInvalidArguments + } + return offset, nil +} + +func encodeWorkloadCursor(offset, digest string) string { + if offset == "" { + return "" + } + return base64.RawURLEncoding.EncodeToString([]byte(digest + ":" + offset)) +} + func normalizePageLimit(limit int) int { if limit <= 0 { return 50 diff --git a/pkg/kubernetes/service_test.go b/pkg/kubernetes/service_test.go index a4b71289..e7b45678 100644 --- a/pkg/kubernetes/service_test.go +++ b/pkg/kubernetes/service_test.go @@ -384,18 +384,28 @@ func TestWorkloadsFiltersCountsAndPaginatesSharedIndex(t *testing.T) { defer server.Close() service := newTestService(t, server.URL, Scope{ClusterID: "cluster-a"}) warmTestInventory(t, service, "shop") + assertCursorOffset := func(cursor, want string) { + t.Helper() + raw, err := base64.RawURLEncoding.DecodeString(cursor) + binding, offset, ok := strings.Cut(string(raw), ":") + if err != nil || !ok || len(binding) != 64 || offset != want { + t.Fatalf("opaque cursor %q does not bind offset %s", cursor, want) + } + } page, err := service.Workloads(t.Context(), WorkloadListOptions{Namespace: "shop", Query: "CHECK", Limit: 1}) - if err != nil || len(page.Items) != 1 || page.Items[0].Kind != "Deployment" || page.Items[0].Name != "checkout-a" || page.Next != "1" || page.Truncated { + if err != nil || len(page.Items) != 1 || page.Items[0].Kind != "Deployment" || page.Items[0].Name != "checkout-a" || page.Truncated { t.Fatalf("first workload page = %#v err=%v", page, err) } + assertCursorOffset(page.Next, "1") wantCounts := map[string]int{"Deployment": 2, "StatefulSet": 0, "DaemonSet": 0, "Job": 0, "CronJob": 0, "Pod": 1} if !reflect.DeepEqual(page.Counts, wantCounts) { t.Fatalf("workload counts = %#v, want %#v", page.Counts, wantCounts) } page, err = service.Workloads(t.Context(), WorkloadListOptions{Namespace: "shop", Query: "check", Limit: 1, Cursor: page.Next}) - if err != nil || len(page.Items) != 1 || page.Items[0].Name != "checkout-b" || page.Next != "2" { + if err != nil || len(page.Items) != 1 || page.Items[0].Name != "checkout-b" { t.Fatalf("second workload page = %#v err=%v", page, err) } + assertCursorOffset(page.Next, "2") page, err = service.Workloads(t.Context(), WorkloadListOptions{Namespace: "shop", Query: "check", Limit: 1, Cursor: page.Next}) if err != nil || len(page.Items) != 1 || page.Items[0].Kind != "Pod" || page.Next != "" || page.Truncated { t.Fatalf("last workload page = %#v err=%v", page, err) diff --git a/pkg/kubernetes/triage.go b/pkg/kubernetes/triage.go index 30408cf7..21eabc70 100644 --- a/pkg/kubernetes/triage.go +++ b/pkg/kubernetes/triage.go @@ -37,6 +37,8 @@ type IssueOptions struct { IncludeChanges bool RootUID string Context string + OccurrenceID string + Center time.Time targetObject *ObjectRef } @@ -74,14 +76,16 @@ type BrokenTotals struct { } type IssuePage struct { - RootContext *PersistedRootContext `json:"root_context,omitempty"` - Items []Issue `json:"items"` - Totals map[string]int `json:"totals"` - Truncated bool `json:"truncated"` - Next string `json:"next,omitempty"` - Partial []PartialFailure `json:"partial_failures,omitempty"` - Sync SyncStatus `json:"sync"` - Broken BrokenTotals `json:"broken"` + SelectedOccurrence *Issue `json:"selected_occurrence,omitempty"` + SelectionStatus string `json:"selection_status,omitempty"` + RootContext *PersistedRootContext `json:"root_context,omitempty"` + Items []Issue `json:"items"` + Totals map[string]int `json:"totals"` + Truncated bool `json:"truncated"` + Next string `json:"next,omitempty"` + Partial []PartialFailure `json:"partial_failures,omitempty"` + Sync SyncStatus `json:"sync"` + Broken BrokenTotals `json:"broken"` } // PersistedRootContext describes captured identity, not a live graph lookup. @@ -121,8 +125,14 @@ func (service *Service) Issues(ctx context.Context, options IssueOptions) (Issue if options.Context != "persisted" || options.RootUID == "" || !safeSegment(options.RootUID) || options.Namespace != "" && !safeSegment(options.Namespace) || !options.Since.IsZero() || !options.Until.IsZero() || options.Cursor != "" || options.Kind != "" || options.Severity != "" || options.Broken { return IssuePage{}, ErrInvalidArguments } + if options.OccurrenceID != "" && (!safeSegment(options.OccurrenceID) || len(options.OccurrenceID) > 253) || !options.Center.IsZero() && (options.Center.Before(time.Unix(1, 0)) || options.Center.After(time.Now())) { + return IssuePage{}, ErrInvalidArguments + } return service.persistedRootContext(ctx, options) } + if options.OccurrenceID != "" || !options.Center.IsZero() { + return IssuePage{}, ErrInvalidArguments + } if !options.Since.IsZero() || !options.Until.IsZero() { if options.Severity != "" && options.Severity != "critical" && options.Severity != "warning" && options.Severity != "info" { return IssuePage{}, ErrInvalidArguments diff --git a/tests/fakekube/inventory_continuity_integration_test.go b/tests/fakekube/inventory_continuity_integration_test.go new file mode 100644 index 00000000..7681e716 --- /dev/null +++ b/tests/fakekube/inventory_continuity_integration_test.go @@ -0,0 +1,949 @@ +//go:build integration + +package fakekube + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/VersusControl/versus-incident/pkg/core" + "github.com/VersusControl/versus-incident/pkg/kubernetes" + "github.com/VersusControl/versus-incident/pkg/storage" +) + +// These tests prove committed Kubernetes inventory continuity across a process +// restart against real durable storage: the file provider on disk and, when +// TEST_POSTGRES_DSN names an isolated database, real PostgreSQL. The Kubernetes +// API is the fakekube server behind a small fault gate, because a real cluster +// cannot hold one LIST open, revoke a credential mid-run or present a second +// physical identity on demand. Every read is traced with safe, typed metadata +// only (scope, error code, sync state, kind status, UIDs of synthetic objects). + +var continuityWorkloadResources = []string{"deployments", "statefulsets", "daemonsets", "jobs", "cronjobs", "pods", "replicasets"} + +// continuityGate wraps the fake API. Authority probes (limit=1) pass a hold so a +// restore can still prove current access while the replacement LIST is pending. +type continuityGate struct { + next http.Handler + mu sync.Mutex + held map[string]bool + release chan struct{} + heldCount int + denyAll int + denyKinds map[string]int + denyClusterL bool + // denyClusterKinds forbids only the cluster-wide LIST of a resource, like a + // credential that may list that kind inside namespaces but not across them. + denyClusterKinds map[string]bool + denyNS map[string]bool +} + +func (gate *continuityGate) ServeHTTP(writer http.ResponseWriter, request *http.Request) { + if strings.HasPrefix(request.URL.Path, "/_fake/") { + gate.next.ServeHTTP(writer, request) + return + } + resource := resourceForPath(request.URL.Path) + isList := request.Method == http.MethodGet && resource != "" && !isItemPath(request.URL.Path) && request.URL.Query().Get("watch") == "" + namespace := namespaceForPath(request.URL.Path) + gate.mu.Lock() + denyAll, denyKind := gate.denyAll, gate.denyKinds[resource] + denyCluster := (gate.denyClusterL || gate.denyClusterKinds[resource]) && isList && namespace == "" && resource != "namespaces" + denyNamespace := isList && namespace != "" && gate.denyNS[namespace] + hold := isList && gate.held[resource] && request.URL.Query().Get("limit") != "1" + release := gate.release + if hold { + gate.heldCount++ + } + gate.mu.Unlock() + switch { + case denyAll != 0: + http.Error(writer, http.StatusText(denyAll), denyAll) + return + case denyKind != 0 && isList: + http.Error(writer, http.StatusText(denyKind), denyKind) + return + case denyCluster || denyNamespace: + http.Error(writer, http.StatusText(http.StatusForbidden), http.StatusForbidden) + return + } + if hold { + select { + case <-release: + case <-request.Context().Done(): + return + } + } + gate.next.ServeHTTP(writer, request) +} + +func (gate *continuityGate) hold(resources ...string) { + gate.mu.Lock() + defer gate.mu.Unlock() + gate.held = map[string]bool{} + for _, resource := range resources { + gate.held[resource] = true + } + gate.release = make(chan struct{}) +} + +func (gate *continuityGate) releaseHolds() { + gate.mu.Lock() + defer gate.mu.Unlock() + if gate.release != nil { + close(gate.release) + } + gate.held, gate.release = map[string]bool{}, nil +} + +func (gate *continuityGate) reset() { + gate.releaseHolds() + gate.mu.Lock() + gate.denyAll, gate.denyKinds, gate.denyClusterL, gate.denyClusterKinds, gate.denyNS = 0, map[string]int{}, false, map[string]bool{}, map[string]bool{} + gate.mu.Unlock() +} + +type continuityCluster struct { + server *Server + gate *continuityGate + url string +} + +func newContinuityCluster(t *testing.T, physicalUID, marker string) *continuityCluster { + t.Helper() + server, err := NewServer(Config{}) + if err != nil { + t.Fatal(err) + } + gate := &continuityGate{next: server, held: map[string]bool{}, denyKinds: map[string]int{}, denyClusterKinds: map[string]bool{}, denyNS: map[string]bool{}} + upstream := httptest.NewServer(gate) + t.Cleanup(func() { gate.reset(); upstream.Close() }) + put := func(resource, namespace, name string, object map[string]any) { + t.Helper() + if err := putObject(server.store, resource, namespace, name, object); err != nil { + t.Fatal(err) + } + } + put("namespaces", "", "kube-system", map[string]any{"apiVersion": "v1", "kind": "Namespace", "metadata": map[string]any{"uid": physicalUID}}) + for _, namespace := range []string{"shop", "payments"} { + put("namespaces", "", namespace, map[string]any{"apiVersion": "v1", "kind": "Namespace", "metadata": map[string]any{"uid": physicalUID + "-" + namespace}}) + } + put("nodes", "", "node-a", map[string]any{"apiVersion": "v1", "kind": "Node", "metadata": map[string]any{"uid": physicalUID + "-node-a"}, "status": map[string]any{"conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}) + cluster := &continuityCluster{server: server, gate: gate, url: upstream.URL} + cluster.deployment("shop", "web", "web-v1") + cluster.deployment("payments", "ledger", "ledger-v1") + cluster.deployment("shop", marker, marker) + put("statefulsets", "shop", "db", map[string]any{"apiVersion": "apps/v1", "kind": "StatefulSet", "metadata": map[string]any{"uid": "db-v1"}, "spec": map[string]any{"replicas": 1}, "status": map[string]any{"replicas": 1, "readyReplicas": 1}}) + put("cronjobs", "payments", "nightly", map[string]any{"apiVersion": "batch/v1", "kind": "CronJob", "metadata": map[string]any{"uid": "nightly-v1"}, "spec": map[string]any{"schedule": "0 1 * * *"}}) + return cluster +} + +// deployment writes a Deployment with one controller-owned ReplicaSet and Pod. +func (cluster *continuityCluster) deployment(namespace, name, uid string) { + labels := map[string]string{"app": name} + store := cluster.server.store + _ = putObject(store, "deployments", namespace, name, map[string]any{"apiVersion": "apps/v1", "kind": "Deployment", "metadata": map[string]any{"uid": uid, "labels": labels, "generation": 1}, "spec": map[string]any{"replicas": 1, "selector": map[string]any{"matchLabels": labels}}, "status": map[string]any{"replicas": 1, "readyReplicas": 1, "availableReplicas": 1, "observedGeneration": 1}}) + _ = putObject(store, "replicasets", namespace, name+"-rs", map[string]any{"apiVersion": "apps/v1", "kind": "ReplicaSet", "metadata": map[string]any{"uid": uid + "-rs", "labels": labels, "ownerReferences": []any{map[string]any{"apiVersion": "apps/v1", "kind": "Deployment", "name": name, "uid": uid, "controller": true}}}, "spec": map[string]any{"replicas": 1, "selector": map[string]any{"matchLabels": labels}}, "status": map[string]any{"replicas": 1, "readyReplicas": 1}}) + _ = putObject(store, "pods", namespace, name+"-0", map[string]any{"apiVersion": "v1", "kind": "Pod", "metadata": map[string]any{"uid": uid + "-pod", "labels": labels, "ownerReferences": []any{map[string]any{"apiVersion": "apps/v1", "kind": "ReplicaSet", "name": name + "-rs", "uid": uid + "-rs", "controller": true}}}, "spec": map[string]any{"nodeName": "node-a", "containers": []any{map[string]any{"name": "app", "image": "example.invalid/" + name + ":v1"}}}, "status": map[string]any{"phase": "Running", "containerStatuses": []any{map[string]any{"name": "app", "ready": true, "restartCount": 0, "state": map[string]any{"running": map[string]any{}}}}}}) +} + +func (cluster *continuityCluster) deleteDeployment(namespace, name string) { + store := cluster.server.store + store.Delete("deployments", namespace, name) + store.Delete("replicasets", namespace, name+"-rs") + store.Delete("pods", namespace, name+"-0") +} + +// continuityStore opens one real durable provider and reopens it like a restart. +type continuityStore struct { + name string + open func(t *testing.T) storage.Provider + tryOpen func(t *testing.T) (storage.Provider, error) + breakS func(t *testing.T) + repair func(t *testing.T) + meta func(t *testing.T, org string) map[string]any +} + +func continuityStores(t *testing.T) []continuityStore { + dir := filepath.Join(t.TempDir(), "file-provider") + stores := []continuityStore{{ + name: "file", + open: func(t *testing.T) storage.Provider { + provider, err := storage.NewFile(storage.FileOptions{DataDir: dir}) + if err != nil { + t.Fatalf("open file provider: %v", err) + } + return provider + }, + tryOpen: func(t *testing.T) (storage.Provider, error) { + return storage.NewFile(storage.FileOptions{DataDir: dir}) + }, + breakS: func(t *testing.T) { + if err := os.Chmod(dir, 0); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) + }, + repair: func(t *testing.T) { _ = os.Chmod(dir, 0o700) }, + meta: func(t *testing.T, org string) map[string]any { + names := []string{} + _ = filepath.WalkDir(filepath.Join(dir), func(path string, entry os.DirEntry, err error) error { + if err == nil && !entry.IsDir() && strings.Contains(path, "inventory") { + rel, _ := filepath.Rel(dir, path) + names = append(names, rel) + } + return nil + }) + return map[string]any{"inventory_blobs": len(names)} + }, + }} + dsn := os.Getenv("TEST_POSTGRES_DSN") + if dsn == "" { + t.Log("postgres provider unavailable: TEST_POSTGRES_DSN not set (blocked, not passed)") + return stores + } + admin, err := sql.Open("pgx", dsn) + if err != nil { + admin = nil + } + stores = append(stores, continuityStore{ + name: "postgres", + open: func(t *testing.T) storage.Provider { + provider, err := storage.NewPostgres(storage.PostgresOptions{DSN: dsn}) + if err != nil { + t.Fatalf("open postgres provider: %v", err) + } + t.Cleanup(func() { provider.Close() }) + return provider + }, + tryOpen: func(t *testing.T) (storage.Provider, error) { + provider, err := storage.NewPostgres(storage.PostgresOptions{DSN: dsn}) + if err == nil { + t.Cleanup(func() { provider.Close() }) + } + return provider, err + }, + breakS: func(t *testing.T) { + db := continuityDB(t, dsn, admin) + if _, err := db.Exec("ALTER TABLE vs_blobs RENAME TO vs_blobs_outage"); err != nil { + t.Fatalf("simulate storage outage: %v", err) + } + t.Cleanup(func() { _, _ = db.Exec("ALTER TABLE IF EXISTS vs_blobs_outage RENAME TO vs_blobs") }) + }, + repair: func(t *testing.T) { + db := continuityDB(t, dsn, admin) + if _, err := db.Exec("ALTER TABLE IF EXISTS vs_blobs_outage RENAME TO vs_blobs"); err != nil { + t.Fatalf("repair storage outage: %v", err) + } + }, + meta: func(t *testing.T, org string) map[string]any { + db := continuityDB(t, dsn, admin) + var count int + _ = db.QueryRow("SELECT count(*) FROM vs_blobs WHERE name LIKE 'kubernetes/inventory/%'").Scan(&count) + return map[string]any{"inventory_blobs": count} + }, + }) + return stores +} + +func continuityDB(t *testing.T, dsn string, admin *sql.DB) *sql.DB { + t.Helper() + if admin != nil { + return admin + } + provider, err := storage.NewPostgres(storage.PostgresOptions{DSN: dsn}) + if err != nil { + t.Fatal(err) + } + return provider.(storage.SQLAccessor).DB() +} + +type continuityProcess struct { + service *kubernetes.Service + registry *kubernetes.ServiceRegistry + cancel context.CancelFunc +} + +func startContinuityProcess(t *testing.T, cluster *continuityCluster, provider storage.Provider, scope kubernetes.Scope) *continuityProcess { + t.Helper() + tokenFile := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(tokenFile, []byte("synthetic-reader-token"), 0o600); err != nil { + t.Fatal(err) + } + client, err := kubernetes.NewClient(kubernetes.Config{Endpoint: cluster.url, TokenFile: tokenFile, AllowLoopbackHTTP: true, Timeout: 5 * time.Second}) + if err != nil { + t.Fatal(err) + } + service := kubernetes.NewService(client, scope, time.Minute) + service.SetChangeStorage(provider) + service.SetScrubber(continuityScrubber{}) + registry := kubernetes.NewServiceRegistry(service) + ctx, cancel := context.WithCancel(context.Background()) + registry.StartCapture(ctx) + process := &continuityProcess{service: service, registry: registry, cancel: cancel} + t.Cleanup(process.stop) + return process +} + +type continuityScrubber struct{} + +func (continuityScrubber) Scrub(value string) string { return value } + +// probeWorkloadLogs opens the UID-bound workload log stream and reports which +// pod incarnations it admitted. Only identities and typed codes are traced. +func probeWorkloadLogs(t *testing.T, process *continuityProcess, store continuityStore, phase, namespace, name, uid string) (string, []string) { + t.Helper() + ctx := core.WithCallerAuthorization(context.Background(), core.CallerAuthorization{Authenticated: true, Permissions: map[core.Permission]bool{core.PermissionInfrastructureView: true}}) + ctx, cancel := context.WithTimeout(ctx, 8*time.Second) + defer cancel() + options := kubernetes.WorkloadLogStreamOptions{Kind: "Deployment", Namespace: namespace, Name: name, UID: uid, Container: "app", TailLines: 5, Timestamps: true} + validate := "" + if err := process.service.ValidateWorkloadLogStream(ctx, options); err != nil { + validate = kubernetes.DiagnoseError(err).Code + if errors.Is(err, kubernetes.ErrNotFound) { + validate = "not_found" + } + } + pods := []string{} + session, err := process.service.OpenWorkloadLogStream(ctx, options) + if err == nil { + seen := map[string]bool{} + var mu sync.Mutex + done := make(chan struct{}) + go func() { + defer close(done) + _ = session.Stream(func(event kubernetes.WorkloadLogStreamEvent) error { + mu.Lock() + defer mu.Unlock() + if event.PodUID != "" && !seen[event.PodUID] { + seen[event.PodUID] = true + pods = append(pods, event.PodUID) + } + for _, source := range event.Sources { + if source.Admitted && source.PodUID != "" && !seen[source.PodUID] { + seen[source.PodUID] = true + pods = append(pods, source.PodUID) + } + } + return nil + }) + }() + select { + case <-done: + case <-time.After(4 * time.Second): + } + session.Close() + <-done + mu.Lock() + pods = append([]string(nil), pods...) + mu.Unlock() + } else if validate == "" { + validate = "open:" + kubernetes.DiagnoseError(err).Code + } + continuityTrace(t, continuityObservation{Phase: phase, Storage: store.name, Namespace: namespace, Kind: "Deployment/" + name + "#" + uid, Outcome: "logs", Code: validate, Rows: pods}) + return validate, pods +} + +func (process *continuityProcess) stop() { + closeCtx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + _ = process.registry.CloseContext(closeCtx) + process.cancel() +} + +type continuityObservation struct { + Test string `json:"test"` + Phase string `json:"phase"` + Storage string `json:"storage"` + Namespace string `json:"namespace"` + Kind string `json:"kind,omitempty"` + Outcome string `json:"outcome"` + Code string `json:"code,omitempty"` + Sync map[string]any `json:"sync,omitempty"` + Rows []string `json:"rows,omitempty"` + Counts map[string]int `json:"counts,omitempty"` + Partial []string `json:"partial,omitempty"` + Health map[string]any `json:"health,omitempty"` + Storage2 map[string]any `json:"storage_meta,omitempty"` + At string `json:"at"` + page kubernetes.WorkloadPage + err error +} + +var continuityTraceMu sync.Mutex + +func continuityTrace(t *testing.T, observation continuityObservation) { + observation.Test, observation.At = t.Name(), time.Now().UTC().Format(time.RFC3339Nano) + encoded, _ := json.Marshal(observation) + t.Log(string(encoded)) + dir := os.Getenv("HARNESS_RESULTS_DIR") + if dir == "" { + return + } + continuityTraceMu.Lock() + defer continuityTraceMu.Unlock() + file, err := os.OpenFile(filepath.Join(dir, "inventory-continuity-trace.jsonl"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return + } + defer file.Close() + _, _ = file.Write(append(encoded, '\n')) +} + +// readWorkloads performs one bounded Workloads read and traces its safe metadata. +func readWorkloads(t *testing.T, process *continuityProcess, store continuityStore, phase, namespace, kind string) continuityObservation { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second) + defer cancel() + page, err := process.service.Workloads(ctx, kubernetes.WorkloadListOptions{Namespace: namespace, Kind: kind, Limit: 200}) + observation := continuityObservation{Phase: phase, Storage: store.name, Namespace: namespace, Kind: kind, page: page, err: err} + if err != nil { + observation.Outcome, observation.Code = "error", kubernetes.DiagnoseError(err).Code + if errors.Is(err, context.DeadlineExceeded) { + observation.Code = "deadline" + } + } else { + observation.Outcome = "ok" + observation.Sync = map[string]any{"state": page.Sync.State, "age_s": page.Sync.AgeSeconds, "partial": page.Sync.Partial, "pending": page.Sync.Pending, "pending_kinds": page.Sync.PendingKinds} + for _, item := range page.Items { + observation.Rows = append(observation.Rows, item.Kind+"/"+item.Namespace+"/"+item.Name+"#"+item.UID) + } + sort.Strings(observation.Rows) + observation.Counts = page.Counts + for _, partial := range page.Partial { + observation.Partial = append(observation.Partial, partial.ResourceID+"="+partial.Class) + } + } + healthCtx, healthCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer healthCancel() + if health, healthErr := process.service.SyncHealth(healthCtx); healthErr == nil { + failing := []string{} + for kind, state := range health.Kinds { + if state.State != "ready" { + failing = append(failing, kind+"="+state.State+":"+state.Error) + } + } + sort.Strings(failing) + if len(failing) > 12 { + failing = append(failing[:12], fmt.Sprintf("+%d more", len(failing)-12)) + } + observation.Health = map[string]any{"state": health.State, "class": health.Class, "first_ready": !health.FirstReadyAt.IsZero(), "not_ready_kinds": failing} + } else { + observation.Health = map[string]any{"error": kubernetes.DiagnoseError(healthErr).Code} + } + continuityTrace(t, observation) + return observation +} + +func (observation continuityObservation) has(row string) bool { + for _, value := range observation.Rows { + if strings.HasPrefix(value, row) { + return true + } + } + return false +} + +func (observation continuityObservation) hasUID(uid string) bool { + for _, value := range observation.Rows { + if strings.HasSuffix(value, "#"+uid) { + return true + } + } + return false +} + +func waitWorkloads(t *testing.T, process *continuityProcess, store continuityStore, phase, namespace, kind string, timeout time.Duration, done func(continuityObservation) bool) (continuityObservation, bool) { + t.Helper() + deadline := time.Now().Add(timeout) + var last continuityObservation + for { + last = readWorkloads(t, process, store, phase, namespace, kind) + if done(last) { + return last, true + } + if time.Now().After(deadline) { + return last, false + } + time.Sleep(time.Second) + } +} + +func continuityScope(store continuityStore, name string) kubernetes.Scope { + suffix := fmt.Sprintf("%s-%s-%d", name, store.name, time.Now().UnixNano()) + return kubernetes.Scope{OrgID: "default", ClusterID: "continuity-" + suffix, CredentialID: "reader-epoch-1"} +} + +func warmProcess(t *testing.T, cluster *continuityCluster, store continuityStore, provider storage.Provider, scope kubernetes.Scope, marker string) *continuityProcess { + t.Helper() + process := startContinuityProcess(t, cluster, provider, scope) + observation, ok := waitWorkloads(t, process, store, "warm", "", "", 90*time.Second, func(o continuityObservation) bool { + return o.err == nil && o.page.Sync.State == "ready" && o.hasUID("web-v1") && o.hasUID("ledger-v1") && o.hasUID(marker) && o.hasUID("nightly-v1") && o.hasUID("db-v1") + }) + if !ok { + t.Fatalf("warm committed publication never became ready: %+v", observation) + } + // Visit the shop namespace so a scoped projection is also committed. + readWorkloads(t, process, store, "warm-shop", "shop", "") + return process +} + +func TestInventoryContinuityRestoreWhileReplacementPending(t *testing.T) { + for _, store := range continuityStores(t) { + t.Run(store.name, func(t *testing.T) { + cluster := newContinuityCluster(t, "physical-a", "only-a") + scope := continuityScope(store, "restore") + first := warmProcess(t, cluster, store, store.open(t), scope, "only-a") + first.stop() + continuityTrace(t, continuityObservation{Phase: "after-warm-stop", Storage: store.name, Outcome: "meta", Storage2: store.meta(t, scope.OrgID)}) + + // Replacement collection is pending and the live object was replaced + // by a same-name object with a new UID. + cluster.gate.hold(continuityWorkloadResources...) + cluster.deleteDeployment("shop", "web") + cluster.deployment("shop", "web", "web-v2") + second := startContinuityProcess(t, cluster, store.open(t), scope) + + shop, ok := waitWorkloads(t, second, store, "restart-pending-shop", "shop", "", 20*time.Second, func(o continuityObservation) bool { return o.err == nil && len(o.Rows) > 0 }) + if !ok { + t.Errorf("restart with pending replacement: shop committed rows unavailable from %s storage; last outcome=%s code=%s sync=%v", store.name, shop.Outcome, shop.Code, shop.Sync) + } else { + if !shop.hasUID("web-v1") || shop.hasUID("web-v2") { + t.Errorf("restored shop projection is not the committed generation: rows=%v", shop.Rows) + } + if shop.has("Deployment/payments/") || shop.has("CronJob/payments/") { + t.Errorf("shop projection borrowed another namespace: rows=%v", shop.Rows) + } + if shop.page.Counts["Deployment"] == 0 { + t.Errorf("restored shop counts claim zero deployments: %v", shop.page.Counts) + } + } + payments := readWorkloads(t, second, store, "restart-pending-first-visit-payments", "payments", "") + if payments.err != nil { + t.Errorf("first visit to payments during pending replacement failed: code=%s", payments.Code) + } else if !payments.hasUID("ledger-v1") || payments.has("Deployment/shop/") || payments.has("StatefulSet/shop/") { + t.Errorf("first payments visit is not its committed projection: rows=%v", payments.Rows) + } + cron := readWorkloads(t, second, store, "restart-pending-first-filter-cronjob", "", "CronJob") + if cron.err != nil { + t.Errorf("first CronJob filter during pending replacement failed: code=%s", cron.Code) + } else if !cron.hasUID("nightly-v1") || len(cron.page.Items) != 1 { + t.Errorf("CronJob filter rows=%v", cron.Rows) + } + unseen := readWorkloads(t, second, store, "restart-pending-uncaptured-namespace", "staging", "") + if unseen.err == nil && len(unseen.Rows) > 0 { + t.Errorf("uncaptured namespace borrowed rows: %v", unseen.Rows) + } + + // The restored generation still names the old UID while the live + // same-name object is a new incarnation. The UID-bound log stream + // must not admit the new incarnation's pod for the old UID. + _, pendingPods := probeWorkloadLogs(t, second, store, "restart-pending-old-uid-logs", "shop", "web", "web-v1") + for _, pod := range pendingPods { + if pod != "web-v1-pod" { + t.Errorf("old workload UID admitted a different pod incarnation during pending replacement: %v", pendingPods) + } + } + cluster.gate.releaseHolds() + replaced, ok := waitWorkloads(t, second, store, "replacement-published-shop", "shop", "", 120*time.Second, func(o continuityObservation) bool { return o.err == nil && o.hasUID("web-v2") }) + if !ok { + t.Errorf("replacement with new UID never published: %+v", replaced.Rows) + } else if replaced.hasUID("web-v1") || replaced.hasUID("web-v1-pod") { + t.Errorf("old UID survived beside same-name replacement: rows=%v", replaced.Rows) + } + oldCode, oldPods := probeWorkloadLogs(t, second, store, "replacement-published-old-uid-logs", "shop", "web", "web-v1") + if oldCode == "" || len(oldPods) > 0 { + t.Errorf("same-name replacement satisfied the old workload UID: code=%q pods=%v", oldCode, oldPods) + } + probeWorkloadLogs(t, second, store, "replacement-published-new-uid-logs", "shop", "web", "web-v2") + global := readWorkloads(t, second, store, "replacement-published-global", "", "Deployment") + if global.hasUID("web-v1") { + t.Errorf("global view kept the replaced UID: rows=%v", global.Rows) + } + + // A complete empty replacement removes every payments workload. + cluster.deleteDeployment("payments", "ledger") + cluster.server.store.Delete("cronjobs", "payments", "nightly") + empty, ok := waitWorkloads(t, second, store, "empty-replacement-payments", "payments", "", 150*time.Second, func(o continuityObservation) bool { + return o.err == nil && len(o.Rows) == 0 && o.page.Sync.State == "ready" + }) + if !ok { + t.Errorf("empty replacement did not remove payments rows: outcome=%s rows=%v sync=%v", empty.Outcome, empty.Rows, empty.Sync) + } + second.stop() + + // Restart again with replacement pending: deleted rows must not resurrect. + cluster.gate.hold(continuityWorkloadResources...) + third := startContinuityProcess(t, cluster, store.open(t), scope) + deadline := time.Now().Add(12 * time.Second) + for time.Now().Before(deadline) { + view := readWorkloads(t, third, store, "restart-after-empty-payments", "payments", "") + if view.hasUID("ledger-v1") || view.hasUID("nightly-v1") { + t.Errorf("deleted payments rows resurrected from storage: %v", view.Rows) + break + } + shopView := readWorkloads(t, third, store, "restart-after-replacement-shop", "shop", "") + if shopView.hasUID("web-v1") { + t.Errorf("replaced UID resurrected from storage: %v", shopView.Rows) + break + } + time.Sleep(2 * time.Second) + } + cluster.gate.releaseHolds() + }) + } +} + +func TestInventoryRestoreFailsClosed(t *testing.T) { + for _, store := range continuityStores(t) { + t.Run(store.name, func(t *testing.T) { + cluster := newContinuityCluster(t, "physical-a", "only-a") + scope := continuityScope(store, "fenced") + warm := warmProcess(t, cluster, store, store.open(t), scope, "only-a") + warm.stop() + probe := func(name string, configure func(), cleanup func(), target *continuityCluster, forbidden ...string) { + t.Run(name, func(t *testing.T) { + configure() + defer cleanup() + process := startContinuityProcess(t, target, store.open(t), scope) + deadline := time.Now().Add(10 * time.Second) + seenError := false + for time.Now().Before(deadline) { + for _, namespace := range []string{"", "shop", "payments"} { + view := readWorkloads(t, process, store, name, namespace, "") + for _, uid := range forbidden { + if view.hasUID(uid) { + t.Errorf("%s replayed protected row %s in namespace %q: rows=%v", name, uid, namespace, view.Rows) + return + } + } + seenError = seenError || view.err != nil + } + time.Sleep(time.Second) + } + if !seenError { + t.Logf("%s: no explicit error observed (reads stayed pending/empty)", name) + } + process.stop() + }) + } + probe("revoked-credential", func() { + cluster.gate.mu.Lock() + cluster.gate.denyAll = http.StatusUnauthorized + cluster.gate.mu.Unlock() + }, cluster.gate.reset, cluster, "web-v1", "ledger-v1", "only-a", "nightly-v1", "db-v1") + probe("whole-denial", func() { + cluster.gate.mu.Lock() + for _, resource := range append(continuityWorkloadResources, "namespaces", "nodes", "events") { + cluster.gate.denyKinds[resource] = http.StatusForbidden + } + cluster.gate.mu.Unlock() + }, cluster.gate.reset, cluster, "web-v1", "ledger-v1", "only-a", "nightly-v1", "db-v1") + probe("namespace-denial", func() { + cluster.gate.mu.Lock() + cluster.gate.denyClusterL, cluster.gate.denyNS = true, map[string]bool{"shop": true} + cluster.gate.mu.Unlock() + cluster.gate.hold("deployments", "statefulsets", "cronjobs", "pods", "replicasets", "jobs", "daemonsets") + }, cluster.gate.reset, cluster, "web-v1", "only-a", "db-v1") + other := newContinuityCluster(t, "physical-b", "only-b") + probe("physical-cluster-mismatch", func() { other.gate.hold(continuityWorkloadResources...) }, other.gate.reset, other, "only-a") + }) + } +} + +func TestInventoryStorageOutageKeepsMemoryAndReportsError(t *testing.T) { + for _, store := range continuityStores(t) { + t.Run(store.name, func(t *testing.T) { + cluster := newContinuityCluster(t, "physical-a", "only-a") + scope := continuityScope(store, "outage") + process := warmProcess(t, cluster, store, store.open(t), scope, "only-a") + baseline := readWorkloads(t, process, store, "warm-before-outage", "", "") + baseClass, _ := baseline.Health["class"].(string) + store.breakS(t) + cluster.deployment("shop", "added", "added-v1") + observed := false + deadline := time.Now().Add(100 * time.Second) + for time.Now().Before(deadline) { + view := readWorkloads(t, process, store, "warm-storage-outage", "", "") + if view.err != nil || !view.hasUID("web-v1") { + t.Errorf("authorized memory publication lost during storage outage: code=%s rows=%v", view.Code, view.Rows) + break + } + class, _ := view.Health["class"].(string) + if view.page.Sync.State != "ready" || len(view.page.Partial) > len(baseline.page.Partial) || class != baseClass { + observed = true + t.Logf("storage outage observable: sync=%v partial=%v health=%v", view.Sync, view.Partial, view.Health) + break + } + time.Sleep(2 * time.Second) + } + if !observed { + t.Errorf("storage outage stayed invisible for 100s: Workloads stayed ready without a storage partial for %s", store.name) + } + cluster.gate.hold(continuityWorkloadResources...) + coldProvider, openErr := store.tryOpen(t) + if openErr != nil { + continuityTrace(t, continuityObservation{Phase: "cold-storage-outage-open", Storage: store.name, Outcome: "error", Code: "storage_open_refused"}) + } else { + cold := startContinuityProcess(t, cluster, coldProvider, continuityScope(store, "outage-cold")) + deadline = time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + view := readWorkloads(t, cold, store, "cold-storage-outage", "", "") + if view.err == nil && view.page.Sync.State == "ready" && len(view.Rows) == 0 { + t.Errorf("cold read during storage outage manufactured ready empty inventory: sync=%v", view.Sync) + break + } + time.Sleep(time.Second) + } + cold.stop() + } + cluster.gate.releaseHolds() + store.repair(t) + recovered, ok := waitWorkloads(t, process, store, "storage-repaired", "", "", 120*time.Second, func(o continuityObservation) bool { + return o.err == nil && o.page.Sync.State == "ready" && o.hasUID("added-v1") + }) + if !ok { + t.Errorf("did not recover after storage repair: %+v", recovered.Sync) + } + }) + } +} + +func TestInventoryOptionalKindPendingOrDenied(t *testing.T) { + for _, store := range continuityStores(t) { + t.Run(store.name, func(t *testing.T) { + cluster := newContinuityCluster(t, "physical-a", "only-a") + cluster.gate.hold("cronjobs") + cluster.gate.mu.Lock() + cluster.gate.denyKinds["daemonsets"] = http.StatusForbidden + cluster.gate.mu.Unlock() + process := startContinuityProcess(t, cluster, store.open(t), continuityScope(store, "optional")) + // No successful LIST means no known count, including when the first + // attempt has finished with a timeout rather than remaining pending. + knownWhileUnobserved := []string{} + sawNeverListedTimeout := false + unobservedCountsUnknown := func(o continuityObservation) { + if o.err != nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + health, err := process.service.SyncHealth(ctx) + if carried, ok := kubernetes.SyncHealthReport(err); ok { + health = carried + } + // Health is read after the page; ObservedAt retains successful + // observations, so zero now also means unobserved on that page. + for _, kind := range []string{"Deployment", "StatefulSet", "DaemonSet", "Job", "CronJob", "Pod"} { + state := health.Kinds[kind] + if count, known := o.page.Counts[kind]; known && state.ObservedAt.IsZero() && len(knownWhileUnobserved) < 12 { + knownWhileUnobserved = append(knownWhileUnobserved, fmt.Sprintf("%s %s ns=%q sync=%s %s=%d state=%s error=%s", o.Storage, o.Phase, o.Namespace, o.page.Sync.State, kind, count, state.State, state.Error)) + } + if kind == "CronJob" && state.ObservedAt.IsZero() && state.State == "error" && state.Error == "timeout" { + for _, partial := range o.page.Partial { + sawNeverListedTimeout = sawNeverListedTimeout || partial.ResourceID == "batch~v1~cronjobs" && partial.Class == "timeout" + } + } + } + } + view, ok := waitWorkloads(t, process, store, "optional-kind-pending-denied", "", "", 60*time.Second, func(o continuityObservation) bool { + unobservedCountsUnknown(o) + return o.err == nil && o.hasUID("web-v1") && o.hasUID("db-v1") && sawNeverListedTimeout + }) + if !ok { + t.Fatalf("one pending and one denied optional kind blocked eligible workloads: code=%s sync=%v", view.Code, view.Sync) + } + denied := false + for _, partial := range view.page.Partial { + denied = denied || partial.ResourceID == "apps~v1~daemonsets" && partial.Class == "forbidden" + } + if !denied { + t.Errorf("denied DaemonSet kind not reported as partial forbidden: %v", view.Partial) + } + pending := view.page.Sync.State != "ready" + for _, id := range view.page.Sync.Pending { + pending = pending || id == "batch~v1~cronjobs" + } + if !pending { + t.Errorf("pending CronJob kind reported as complete ready: sync=%v counts=%v", view.Sync, view.Counts) + } + if view.hasUID("nightly-v1") { + t.Errorf("held CronJob rows appeared before its LIST completed") + } + shop := readWorkloads(t, process, store, "optional-kind-first-namespace", "shop", "") + if shop.err != nil || !shop.hasUID("web-v1") { + t.Errorf("first shop visit with optional kinds pending/denied: code=%s rows=%v", shop.Code, shop.Rows) + } + unobservedCountsUnknown(shop) + if len(knownWhileUnobserved) > 0 { + t.Errorf("never-listed workload kinds reported as known counts instead of unknown: %v", knownWhileUnobserved) + } + cluster.gate.releaseHolds() + process.stop() + }) + } +} + +// A Workloads cursor issued under one committed generation must not silently +// page a different generation: a replacement that inserts or removes rows +// before the cursor would otherwise duplicate or skip rows as a success. +func TestWorkloadCursorAcrossReplacementGeneration(t *testing.T) { + for _, store := range continuityStores(t) { + t.Run(store.name, func(t *testing.T) { + cluster := newContinuityCluster(t, "physical-a", "only-a") + process := warmProcess(t, cluster, store, store.open(t), continuityScope(store, "cursor"), "only-a") + ctx := context.Background() + first, err := process.service.Workloads(ctx, kubernetes.WorkloadListOptions{Kind: "Deployment", Limit: 1}) + if err != nil || len(first.Items) != 1 || first.Next == "" { + t.Fatalf("first page: %v %+v", err, first) + } + firstRow := first.Items[0].Namespace + "/" + first.Items[0].Name + following, err := process.service.Workloads(ctx, kubernetes.WorkloadListOptions{Kind: "Deployment", Limit: 1, Cursor: first.Next}) + if err != nil || len(following.Items) != 1 { + t.Fatalf("second page before replacement: %v", err) + } + expectedNext := following.Items[0].Namespace + "/" + following.Items[0].Name + continuityTrace(t, continuityObservation{Phase: "cursor-issued", Storage: store.name, Kind: "Deployment", Outcome: "ok", Rows: []string{firstRow, expectedNext}}) + // A routine resync that lists the same Deployments again must not expire + // the issued cursor: the unchanged generation still pages to its successor. + baseline := kindObservedAt(t, process, "Deployment") + resynced := false + for deadline := time.Now().Add(120 * time.Second); time.Now().Before(deadline); time.Sleep(time.Second) { + if resynced = kindObservedAt(t, process, "Deployment").After(baseline); resynced { + break + } + } + if !resynced { + t.Errorf("no routine Deployment resync observed within 120s; unchanged-generation continuation not exercised") + } else { + again, err := process.service.Workloads(ctx, kubernetes.WorkloadListOptions{Kind: "Deployment", Limit: 1, Cursor: first.Next}) + observation := continuityObservation{Phase: "cursor-reused-after-unchanged-resync", Storage: store.name, Kind: "Deployment", Outcome: "ok"} + if err != nil { + observation.Outcome, observation.Code = "error", kubernetes.DiagnoseError(err).Code + } + for _, item := range again.Items { + observation.Rows = append(observation.Rows, item.Namespace+"/"+item.Name) + } + continuityTrace(t, observation) + if err != nil || len(observation.Rows) != 1 || observation.Rows[0] != expectedNext { + t.Errorf("unchanged resync expired or moved the cursor: err=%v rows=%v want %s", err, observation.Rows, expectedNext) + } + } + // Replacement inserts a row that sorts before the issued cursor. + cluster.deployment("payments", "aaa-inserted", "aaa-inserted-v1") + if _, ok := waitWorkloads(t, process, store, "cursor-replacement", "", "Deployment", 90*time.Second, func(o continuityObservation) bool { return o.err == nil && o.hasUID("aaa-inserted-v1") }); !ok { + t.Fatal("replacement with inserted row never published") + } + stale, err := process.service.Workloads(ctx, kubernetes.WorkloadListOptions{Kind: "Deployment", Limit: 1, Cursor: first.Next}) + observation := continuityObservation{Phase: "cursor-reused-after-replacement", Storage: store.name, Kind: "Deployment", Outcome: "ok"} + if err != nil { + observation.Outcome, observation.Code = "error", kubernetes.DiagnoseError(err).Code + } else { + for _, item := range stale.Items { + observation.Rows = append(observation.Rows, item.Namespace+"/"+item.Name) + } + } + continuityTrace(t, observation) + if !errors.Is(err, kubernetes.ErrInvalidArguments) { + t.Errorf("stale cursor %q must return typed invalidation: err=%v rows=%v, page one was %s and its successor was %s", first.Next, err, observation.Rows, firstRow, expectedNext) + } + }) + } +} + +// kindObservedAt reports when the last successful LIST of one kind started. +func kindObservedAt(t *testing.T, process *continuityProcess, kind string) time.Time { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + health, err := process.service.SyncHealth(ctx) + if carried, ok := kubernetes.SyncHealthReport(err); ok { + health = carried + } + return health.Kinds[kind].ObservedAt +} + +// A namespace-scoped snapshot committed while the cluster-wide Pod LIST was +// forbidden must not hide a later cluster-wide Pod timeout. When the global +// LIST is allowed again but times out, a scoped read reports that timeout +// instead of presenting the older scoped rows as a complete, current view. +func TestScopedSnapshotDoesNotHideNewerGlobalTimeout(t *testing.T) { + for _, store := range continuityStores(t) { + t.Run(store.name, func(t *testing.T) { + cluster := newContinuityCluster(t, "physical-a", "only-a") + process := warmProcess(t, cluster, store, store.open(t), continuityScope(store, "scoped-timeout"), "only-a") + podState := func() (string, string, time.Time) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + health, err := process.service.SyncHealth(ctx) + if carried, ok := kubernetes.SyncHealthReport(err); ok { + health = carried + } + state := health.Kinds["Pod"] + return state.State, state.Error, state.LastAttemptAt + } + podPartial := func(o continuityObservation) string { + for _, partial := range o.page.Partial { + if partial.ResourceID == "core~v1~pods" { + return partial.Class + } + } + return "" + } + cluster.gate.mu.Lock() + cluster.gate.denyClusterKinds["pods"] = true + cluster.gate.mu.Unlock() + scoped, ok := waitWorkloads(t, process, store, "scoped-pods-under-cluster-denial", "shop", "Pod", 150*time.Second, func(o continuityObservation) bool { + // The worker proves namespace authority and reports the cluster-wide + // limitation as forbidden or namespace_scope; either admits scoped rows. + _, class, _ := podState() + return (class == "forbidden" || class == "namespace_scope") && o.err == nil && o.hasUID("web-v1-pod") && podPartial(o) == "" + }) + if !ok { + t.Fatalf("namespace-scoped Pod snapshot never replaced the cluster-wide denial: code=%s rows=%v partial=%v", scoped.Code, scoped.Rows, scoped.Partial) + } + scopedAt := time.Now() + cluster.gate.mu.Lock() + cluster.gate.denyClusterKinds = map[string]bool{} + cluster.gate.mu.Unlock() + cluster.gate.hold("pods") + defer cluster.gate.releaseHolds() + globalClass := "" + for deadline := time.Now().Add(150 * time.Second); time.Now().Before(deadline); time.Sleep(time.Second) { + state, class, attempted := podState() + if state == "error" && class != "" && class != "forbidden" && class != "namespace_scope" && attempted.After(scopedAt) { + globalClass = class + break + } + } + continuityTrace(t, continuityObservation{Phase: "global-pods-after-scoped", Storage: store.name, Kind: "Pod", Outcome: "health", Code: globalClass}) + if globalClass == "" { + t.Fatalf("cluster-wide Pod LIST never recorded a newer non-permission failure; scenario not exercised") + } + global := readWorkloads(t, process, store, "global-after-newer-timeout", "", "Pod") + if global.err == nil && podPartial(global) != globalClass { + t.Errorf("cluster-wide read hid the newer Pod %s: partial=%v", globalClass, global.Partial) + } + view := readWorkloads(t, process, store, "scoped-after-newer-global-timeout", "shop", "Pod") + if view.err != nil { + t.Errorf("non-permission Pod %s blocked the scoped read: code=%s", globalClass, view.Code) + } else if podPartial(view) != globalClass { + t.Errorf("older scoped snapshot concealed the newer cluster-wide Pod %s: sync=%v partial=%v rows=%v", globalClass, view.Sync, view.Partial, view.Rows) + } + cluster.gate.releaseHolds() + process.stop() + }) + } +} diff --git a/ui/src/lib/api.ts b/ui/src/lib/api.ts index ebf3ea9c..5994eeff 100644 --- a/ui/src/lib/api.ts +++ b/ui/src/lib/api.ts @@ -2333,6 +2333,8 @@ export interface KubernetesIssue { } export interface KubernetesIssuePage { + selected_occurrence?: KubernetesIssue; + selection_status?: "not_requested" | "not_found" | "resolved" | "ambiguous" | "partial"; /** Persisted UID identity, independent of the selected history window and its totals. */ root_context?: { root: KubernetesObjectRef; occurrence: KubernetesIssue; retained: true }; items: KubernetesIssue[]; @@ -2959,9 +2961,11 @@ export interface KubernetesBreakLogPage { export function createKubernetesApi(cluster?: string) { const request: typeof unscopedRequest = (path, options) => unscopedRequest(kubernetesClusterPath(path, cluster), options); - async function historyPage(path: string, original: URLSearchParams, signal?: AbortSignal): Promise { + async function historyPage(path: string, original: URLSearchParams, signal?: AbortSignal, pageMode?: "single"): Promise { const query = new URLSearchParams(original); if (query.has("limit")) query.set("limit", String(Math.min(500, Math.max(1, Number(query.get("limit")) || 100)))); + // Focused scans own one shared HTTP request budget, including empty pages. + if (pageMode === "single") return request(`${path}?${query}`, { signal }); const seen = new Set(); const gaps: unknown[] = []; let restarted = false; @@ -3008,26 +3012,26 @@ export function createKubernetesApi(cluster?: string) { kubernetesSaveAgentSettings: (settings: KubernetesAgentSettingsInput) => request("/api/admin/kubernetes/agent-settings", { method: "PUT", body: JSON.stringify({ learning: settings.learning, break_logs: settings.break_logs }) }), kubernetesTestLink: (source: string, fields: KubernetesLogFields) => request("/api/admin/kubernetes/agent-settings/test-link", { method: "POST", body: JSON.stringify({ source, fields }) }), kubernetesOverviewGraph: () => request("/api/admin/kubernetes/graph/overview"), - kubernetesIssues: (options: { since?: string; until?: string; namespace?: string; severity?: string; kind?: string; limit?: number; cursor?: string; broken?: boolean; include?: "changes"; root_uid?: string; context?: "persisted" } = {}, signal?: AbortSignal) => { + kubernetesIssues: (options: { since?: string; until?: string; namespace?: string; severity?: string; kind?: string; limit?: number; cursor?: string; broken?: boolean; include?: "changes"; root_uid?: string; context?: "persisted"; occurrence_id?: string; center?: string } = {}, signal?: AbortSignal) => { const query = new URLSearchParams(); for (const [key, value] of Object.entries(options)) { if (value !== undefined && value !== "") query.set(key, String(value)); } return historyPage("/api/admin/kubernetes/issues", query, signal); }, - kubernetesChanges: (options: { since?: string; until?: string; namespace?: string; kind?: string; name?: string; root_uid?: string; type?: KubernetesChangeType; limit?: number; cursor?: string } = {}, signal?: AbortSignal) => { + kubernetesChanges: (options: { since?: string; until?: string; namespace?: string; kind?: string; name?: string; root_uid?: string; type?: KubernetesChangeType; limit?: number; cursor?: string } = {}, signal?: AbortSignal, pageMode?: "single") => { const query = new URLSearchParams(); for (const [key, value] of Object.entries(options)) { if (value !== undefined && value !== "") query.set(key, String(value)); } - return historyPage("/api/admin/kubernetes/changes", query, signal); + return historyPage("/api/admin/kubernetes/changes", query, signal, pageMode); }, - kubernetesCrashes: (options: KubernetesCrashQuery = {}, signal?: AbortSignal) => { + kubernetesCrashes: (options: KubernetesCrashQuery = {}, signal?: AbortSignal, pageMode?: "single") => { const query = new URLSearchParams(); for (const [key, value] of Object.entries(options)) { if (value !== undefined && value !== "") query.set(key, String(value)); } - return historyPage("/api/admin/kubernetes/crashes", query, signal); + return historyPage("/api/admin/kubernetes/crashes", query, signal, pageMode); }, kubernetesTimeline: (options: KubernetesTimelineQuery, signal?: AbortSignal) => { const query = new URLSearchParams(); @@ -3079,12 +3083,12 @@ export function createKubernetesApi(cluster?: string) { kubernetesUsage: (namespace = "") => request(`/api/admin/kubernetes/usage?namespace=${encodeURIComponent(namespace)}`), kubernetesUsageHistory: (namespace: string, pod: string, signal?: AbortSignal) => request(`/api/admin/kubernetes/usage-history?${new URLSearchParams({ namespace, pod })}`, { signal }), - kubernetesWorkloads: async (options: { namespace?: string; kind?: string; q?: string; limit?: number; cursor?: string } = {}) => { + kubernetesWorkloads: async (options: { namespace?: string; kind?: string; q?: string; limit?: number; cursor?: string } = {}, signal?: AbortSignal) => { const query = new URLSearchParams(); for (const [key, value] of Object.entries(options)) { if (value !== undefined && value !== "") query.set(key, String(value)); } - const page = await request & { items: KubernetesResource[] | null }>(`/api/admin/kubernetes/workloads?${query}`); + const page = await request & { items: KubernetesResource[] | null }>(`/api/admin/kubernetes/workloads?${query}`, { signal }); return { ...page, items: Array.isArray(page.items) ? page.items : [] }; }, kubernetesWorkload: (kind: string, namespace: string, name: string) => diff --git a/ui/src/lib/kubernetesApi.test.ts b/ui/src/lib/kubernetesApi.test.ts index e15c8892..36935504 100644 --- a/ui/src/lib/kubernetesApi.test.ts +++ b/ui/src/lib/kubernetesApi.test.ts @@ -1,6 +1,7 @@ // @vitest-environment jsdom import { afterEach, describe, expect, it, vi } from "vitest"; import { api, clearSecret, createKubernetesApi, kubernetesClusterPath, setSecret } from "./api"; +import { focusedChanges } from "./kubernetesFocusedHistory"; afterEach(() => { vi.restoreAllMocks(); @@ -8,6 +9,39 @@ afterEach(() => { }); describe("Kubernetes resource API", () => { + it("carries selected occurrence and center separately from date-free retained root identity", async () => { + const selected = { occurrence_id: "selected", root: { uid: "root" }, broke_at: "2026-10-14T12:00:00Z" }; + const fetch = vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response(JSON.stringify({ items: [], selected_occurrence: selected, selection_status: "resolved" }))); + const result = await createKubernetesApi("east").kubernetesIssues({ context: "persisted", root_uid: "root", namespace: "shop", occurrence_id: "selected", center: selected.broke_at }); + expect(Object.fromEntries(new URL(String(fetch.mock.calls[0][0]), "http://localhost").searchParams)).toEqual({ + context: "persisted", root_uid: "root", namespace: "shop", occurrence_id: "selected", center: selected.broke_at, cluster: "east", + }); + expect(result.selected_occurrence).toEqual(selected); + expect(result.selection_status).toBe("resolved"); + }); + + it.each(["changes", "crashes"] as const)("returns one raw focused %s page including empty continuations, without hidden paging or cursor restart", async route => { + const fetch = vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response(JSON.stringify({ items: [], next: "opaque-next", sync: { state: "ready", age_s: 0, partial: false } }))); + const client = createKubernetesApi("east"), signal = new AbortController().signal; + const result = await client[route === "changes" ? "kubernetesChanges" : "kubernetesCrashes"]({ root_uid: "root", cursor: "position", limit: 100 }, signal, "single"); + expect(fetch).toHaveBeenCalledTimes(1); + expect(result.next).toBe("opaque-next"); + expect(fetch.mock.calls[0][1]?.signal).toBe(signal); + fetch.mockReset().mockResolvedValue(new Response(JSON.stringify({ error: "Invalid query" }), { status: 400 })); + await expect(client[route === "changes" ? "kubernetesChanges" : "kubernetesCrashes"]({ root_uid: "root", cursor: "expired" }, signal, "single")).rejects.toMatchObject({ status: 400 }); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("enforces the focused scan's actual HTTP request budget even on an empty history", async () => { + let page = 0; + const fetch = vi.spyOn(globalThis, "fetch").mockImplementation(async () => new Response(JSON.stringify({ items: [], next: `cursor-${++page}`, sync: { state: "ready", age_s: 0, partial: false } }))); + const signal = new AbortController().signal; + const client = createKubernetesApi("east"); + const result = await focusedChanges(cursor => client.kubernetesChanges({ root_uid: "root", cursor, limit: 100 }, signal, "single"), signal); + expect(fetch).toHaveBeenCalledTimes(5); + expect(result.incomplete_range).toBe(true); + expect(result.next).toBe("cursor-5"); + }); it("fetches persisted UID context independently of history dates and keeps its envelope separate", async () => { const root = { kind: "Deployment", namespace: "shop", name: "retained-root", uid: "root-uid" }; const page = { items: [], totals: {}, truncated: false, sync: { state: "ready", age_s: 0, partial: false }, root_context: { root, occurrence: { root, rule: "resource.context", severity: "info", count: 0 }, retained: true } }; diff --git a/ui/src/lib/kubernetesContinuity.test.ts b/ui/src/lib/kubernetesContinuity.test.ts index 2a08dd2e..8e45b092 100644 --- a/ui/src/lib/kubernetesContinuity.test.ts +++ b/ui/src/lib/kubernetesContinuity.test.ts @@ -12,6 +12,53 @@ const warming = { items: [], broken: { total: 0 }, allocatable_cpu: undefined, o afterEach(cleanup); describe("Kubernetes publication continuity", () => { + it("requires published-cluster and committed projection proof for an initial partial timeout", () => { + const partial = { items: [{ kind: "Deployment", namespace: "shop", name: "checkout" }], counts: { Deployment: 1, Job: 0 }, sync: { state: "error", partial: true }, + partial_failures: [{ resource_id: "batch~v1~cronjobs", class: "timeout" }] }; + expect(hasKubernetesPublication(partial, committed.observed_at)).toBe(true); + expect(hasKubernetesPublication({ ...partial, items: [], counts: { Deployment: 0 } }, committed.observed_at)).toBe(true); + for (const value of [ + partial, + { ...partial, counts: {} }, + { ...partial, counts: { Unknown: 1 } }, + { ...partial, counts: { Deployment: -1 } }, + { ...partial, items: ["untrusted"] }, + { ...partial, items: [{ kind: "CronJob", name: "unobserved", namespace: "shop" }] }, + { ...partial, sync: { state: "syncing", partial: true }, items: [] }, + { ...partial, sync: { state: "error", partial: false } }, + { ...partial, partial_failures: [] }, + ...["forbidden", "unauthorized", "namespace_scope", "credential_unavailable", "unknown"].map(className => + ({ ...partial, partial_failures: [{ resource_id: "batch~v1~cronjobs", class: className }] })), + ]) expect(hasKubernetesPublication(value, value === partial ? undefined : committed.observed_at)).toBe(false); + expect(hasKubernetesPublication(partial, "invalid")).toBe(false); + }); + + it("admits only the currently scoped first partial response and hides permission denial", async () => { + const cache = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + const partial = { items: [{ kind: "Deployment", namespace: "shop", name: "checkout" }], counts: { Deployment: 1, Job: 0 }, sync: { state: "error", partial: true }, + partial_failures: [{ resource_id: "batch~v1~cronjobs", class: "timeout" }] }; + let reply: typeof partial | Error = partial; + const wrapper = ({ children }: { children: ReactNode }) => createElement(QueryClientProvider, { client: cache }, children); + const { result, rerender } = renderHook(({ scope, proof }: { scope: string; proof?: string }) => + useKubernetesQuery({ queryKey: ["kubernetes-workloads", scope], queryFn: async () => { + if (reply instanceof Error) throw reply; + return reply; + } }, proof), { wrapper, initialProps: { scope: "shop", proof: undefined as string | undefined } }); + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(result.current.data).toBeUndefined(); + rerender({ scope: "shop", proof: committed.observed_at }); + expect(result.current.data?.counts).toEqual({ Deployment: 1, Job: 0 }); + rerender({ scope: "other", proof: undefined }); + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(result.current.data).toBeUndefined(); + rerender({ scope: "shop", proof: committed.observed_at }); + reply = new ApiError(403, "Permission denied"); + await act(async () => { await result.current.refetch(); }); + await waitFor(() => expect(result.current.data).toBeUndefined()); + expect(cache.getQueryData(["kubernetes", "_", "kubernetes-workloads", "shop"])).toBeUndefined(); + cache.clear(); + }); + it("preserves committed rows, capacity and observation time across repeated successful warming and failed publications", () => { const refreshing = retainKubernetesPublication(committed, warming); const failed = retainKubernetesPublication(refreshing, { ...warming, sync: { state: "error", age_s: 900, partial: true } }); @@ -74,6 +121,14 @@ describe("Kubernetes publication continuity", () => { expect(kubernetesQueryUsesKinds(["kubernetes-graph"], undefined, new Set())).toBe(false); }); + it("fences workload counts and unfinished mixed workload projections even when the displayed page has no denied-kind rows", () => { + const denied = new Set(["Pod"]); + expect(kubernetesQueryUsesKinds(["kubernetes-workloads", "shop", "", "Deployment"], undefined, denied)).toBe(true); + expect(kubernetesQueryUsesKinds(["kubernetes-workloads"], { items: [{ kind: "Deployment" }], counts: { Pod: 9, Deployment: 1 } }, denied)).toBe(true); + expect(kubernetesQueryUsesKinds(["kubernetes-summary"], { counts: { Pod: 0 } }, denied)).toBe(true); + expect(kubernetesQueryUsesKinds(["kubernetes-workloads"], undefined, new Set(["Secret"]))).toBe(false); + }); + it("rejects a malformed empty sync envelope without inventing a publication", () => { const malformed = { ...warming, sync: { ...warming.sync, state: "" } }; expect(hasKubernetesPublication(malformed)).toBe(false); diff --git a/ui/src/lib/kubernetesContinuity.ts b/ui/src/lib/kubernetesContinuity.ts index 084de05c..1ee1f74a 100644 --- a/ui/src/lib/kubernetesContinuity.ts +++ b/ui/src/lib/kubernetesContinuity.ts @@ -41,12 +41,17 @@ export function kubernetesQueryUsesKinds(parts: readonly unknown[], value: unkno const name = kind.toLowerCase(), candidate = value.toLowerCase(); return candidate === name || candidate.endsWith(`~${name}s`) || candidate.endsWith(`~${name}es`); }); + // Workloads publishes counts for every workload kind, even when this page's + // rows are filtered to another kind. Its in-flight dependency set is known. + if (parts[0] === "kubernetes-workloads" + && ["Pod", "Deployment", "StatefulSet", "DaemonSet", "Job", "CronJob"].some(matches)) return true; const seen = new WeakSet(); const visit = (value: unknown): boolean => { if (!value || typeof value !== "object" || seen.has(value)) return false; seen.add(value); if (Array.isArray(value)) return value.some(visit); const row = value as Record; + if (Object.keys(object(row.counts) ?? {}).some(matches)) return true; return [row.kind, row.root_kind, row.resource_id].some(matches) || Object.values(row).some(visit); }; return parts.some(part => typeof part === "string" && part.includes("~") && matches(part) || visit(part)) || visit(value); @@ -88,10 +93,33 @@ function hasProjectedObservation(value: unknown): boolean { && [projection.nodes, projection.pods, projection.workloads].some((count) => typeof count === "number" && count > 0))); } -export function hasKubernetesPublication(value: unknown): boolean { +export function hasKubernetesPublication(value: unknown, firstReadyAt?: string): boolean { const snapshot = value as Snapshot | undefined; if (snapshot?.[published]) return true; if (snapshot?.sync && (!snapshot.sync.state || snapshot.sync.state === "syncing")) return hasProjectedObservation(value); + if (snapshot?.sync?.state === "error") { + const projection = object(value); + const firstReady = Date.parse(firstReadyAt ?? ""); + const counts = object(projection?.counts); + const workloadKinds = new Set(["Deployment", "StatefulSet", "DaemonSet", "Job", "CronJob", "Pod"]); + const knownCount = (kind: string) => typeof counts?.[kind] === "number" + && Number.isSafeInteger(counts[kind]) && counts[kind] >= 0; + // Workloads has no observation timestamp. Its count keys are emitted only + // for authorized successful observations, including empty ones. Together + // with current scoped cluster authority they prove a committed projection, + // not an arbitrary error or a cold warming response. + return Boolean(Number.isFinite(firstReady) + && snapshot.sync.partial === true && Array.isArray(projection?.items) + && counts && Object.keys(counts).length && Object.keys(counts).every(kind => workloadKinds.has(kind) && knownCount(kind)) + && projection.items.every(item => { + const row = object(item); + return typeof row?.kind === "string" && knownCount(row.kind) + && typeof row.name === "string" && row.name.length > 0 && typeof row.namespace === "string"; + }) + && snapshot.partial_failures?.length && snapshot.partial_failures.every(failure => + kubernetesQueryUsesKinds([object(failure)?.resource_id], undefined, workloadKinds) + && ["timeout", "unavailable"].includes(failure.class ?? ""))); + } return snapshot?.sync?.state !== "error"; } diff --git a/ui/src/lib/kubernetesFocusedHistory.adversarial.test.ts b/ui/src/lib/kubernetesFocusedHistory.adversarial.test.ts new file mode 100644 index 00000000..b5004b98 --- /dev/null +++ b/ui/src/lib/kubernetesFocusedHistory.adversarial.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it, vi } from "vitest"; +import { ApiError, type KubernetesCrashEvent, type KubernetesCrashPage } from "./api"; +import { focusedChanges, focusedCrashes, focusedHistoryRequests } from "./kubernetesFocusedHistory"; + +const from = "2026-10-14T12:00:00Z"; +const to = "2026-10-14T12:10:00Z"; +const sync = { state: "ready", age_s: 0, partial: false }; +const event: KubernetesCrashEvent = { id: "termination", at: from, sequence: 1, cluster: "east", namespace: "shop", pod: "api-0", pod_uid: "pod", container: "api", init: false, root_uid: "root", root_kind: "StatefulSet", root_name: "api", observed_at: from, type: "termination", time_source: "finished_at", restart_count: 1 }; +const page: KubernetesCrashPage = { items: [event], since: from, until: to, retention_from: from, retention_basis: "capture", started_at: from, gaps: [], capture_gaps: [], coverage: { Pod: { from, to } }, limitations: [], capture: { mode: "continuous", behind: false }, sync }; + +describe("Focused history adversarial boundaries", () => { + it("does not publish an inverted coverage interval when admitted pages have disjoint coverage", async () => { + const fetch = vi.fn<(cursor?: string) => Promise>() + .mockResolvedValueOnce({ ...page, next: "next" }) + .mockResolvedValueOnce({ ...page, coverage: { Pod: { from: "2026-10-14T12:20:00Z", to: "2026-10-14T12:30:00Z" } } }); + const result = await focusedCrashes(fetch, new AbortController().signal); + expect(fetch).toHaveBeenCalledTimes(2); + expect(result.coverage).toEqual({}); + expect(result.items).toEqual([event]); + }); + + it.each([ + { from: "invalid", to }, + { from: to, to: from }, + ])("drops malformed coverage on a single admitted page: %j", async coverage => { + const result = await focusedCrashes(async () => ({ ...page, coverage: { Pod: coverage } }), new AbortController().signal); + expect(result.coverage).toEqual({}); + expect(result.items).toEqual([event]); + }); + + it("retains only the common covered interval without manufacturing a union", async () => { + const fetch = vi.fn<(cursor?: string) => Promise>() + .mockResolvedValueOnce({ ...page, next: "next" }) + .mockResolvedValueOnce({ ...page, coverage: { Pod: { from: "2026-10-14T12:05:00Z", to: "2026-10-14T12:15:00Z" } } }); + const result = await focusedCrashes(fetch, new AbortController().signal); + expect(result.coverage).toEqual({ Pod: { from: "2026-10-14T12:05:00.000Z", to: "2026-10-14T12:10:00.000Z" } }); + expect(result.items).toEqual([event]); + }); + + it("detects a multi-token crash continuation cycle through empty pages without erasing gaps", async () => { + const gap = { from, to }; + const fetch = vi.fn<(cursor?: string) => Promise>() + .mockResolvedValueOnce({ ...page, next: "a+/=" }) + .mockResolvedValueOnce({ ...page, items: [], next: "b+/=", capture_gaps: [gap] }) + .mockResolvedValueOnce({ ...page, items: [], next: "a+/=" }); + const result = await focusedCrashes(fetch, new AbortController().signal); + expect(fetch.mock.calls.map(([cursor]) => cursor)).toEqual([undefined, "a+/=", "b+/="]); + expect(result).toMatchObject({ items: [event], incomplete_range: true, capture_gaps: [gap] }); + }); + + it.each([250, 251])("distinguishes the crash display boundary at %i unique observations", async (count) => { + const fetch = vi.fn(async () => ({ ...page, items: Array.from({ length: count }, (_, i) => ({ ...event, id: String(i) })) })); + const result = await focusedCrashes(fetch, new AbortController().signal); + expect(result.items).toHaveLength(Math.min(count, 250)); + expect(result.crash_count).toBe(Math.min(count, 250)); + expect(result.incomplete_range).toBe(count > 250); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("exhausts the crash request budget even when every intermediate page is empty", async () => { + const fetch = vi.fn(async (cursor?: string) => ({ ...page, items: [], next: `${cursor ?? ""}x` })); + const result = await focusedCrashes(fetch, new AbortController().signal); + expect(fetch).toHaveBeenCalledTimes(focusedHistoryRequests); + expect(result).toMatchObject({ items: [], crash_count: 0, incomplete_range: true }); + }); + + it("propagates typed credential loss on a later successful chain rather than retaining authorized-looking crashes", async () => { + const fetch = vi.fn<(cursor?: string) => Promise>() + .mockResolvedValueOnce({ ...page, next: "next" }) + .mockRejectedValueOnce(new ApiError(503, "credential retired", { code: "credential_unavailable", retryable: false })); + await expect(focusedCrashes(fetch, new AbortController().signal)).rejects.toMatchObject({ status: 503 }); + expect(fetch).toHaveBeenCalledTimes(2); + }); + + it("keeps truncation from an empty terminal changes page instead of claiming full evidence", async () => { + const fetch = vi.fn() + .mockResolvedValueOnce({ items: [], next: "next", sync }) + .mockResolvedValueOnce({ items: [], truncated: true, sync }); + const result = await focusedChanges(fetch, new AbortController().signal); + expect(fetch).toHaveBeenCalledTimes(2); + expect(result.truncated).toBe(true); + }); + + it("rejects an obsolete second-page callback after a replacement root has already loaded", async () => { + const controller = new AbortController(); + let finish!: (value: KubernetesCrashPage) => void; + const oldFetch = vi.fn<(cursor?: string) => Promise>() + .mockResolvedValueOnce({ ...page, next: "old-next" }) + .mockImplementationOnce(() => new Promise(resolve => { finish = resolve; })); + const obsolete = focusedCrashes(oldFetch, controller.signal); + const rejection = expect(obsolete).rejects.toMatchObject({ name: "AbortError" }); + await vi.waitFor(() => expect(oldFetch).toHaveBeenCalledTimes(2)); + controller.abort(); + const replacement = { ...event, id: "replacement-event", root_uid: "replacement-root", namespace: "other" }; + const newFetch = vi.fn(async () => ({ ...page, items: [replacement] })); + const current = await focusedCrashes(newFetch, new AbortController().signal); + finish({ ...page, next: "must-not-read", items: [event] }); + await rejection; + expect(oldFetch).toHaveBeenCalledTimes(2); + expect(current.items).toEqual([replacement]); + expect(newFetch).toHaveBeenCalledTimes(1); + }); +}); diff --git a/ui/src/lib/kubernetesFocusedHistory.test.ts b/ui/src/lib/kubernetesFocusedHistory.test.ts new file mode 100644 index 00000000..85e292a7 --- /dev/null +++ b/ui/src/lib/kubernetesFocusedHistory.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it, vi } from "vitest"; +import { ApiError, type KubernetesChange, type KubernetesChangesPage, type KubernetesCrashEvent, type KubernetesCrashPage } from "./api"; +import { focusedChanges, focusedCrashes, focusedHistoryRequests } from "./kubernetesFocusedHistory"; + +const sync = { state: "ready", age_s: 0, partial: false }; +const change: KubernetesChange = { id: "image", at: "2026-10-14T12:00:00Z", cluster: "east", namespace: "shop", kind: "Deployment", name: "checkout", uid: "root", type: "image_changed", source: "capture" }; +const event: KubernetesCrashEvent = { id: "crash", at: change.at, sequence: 1, cluster: "east", namespace: "shop", pod: "checkout", pod_uid: "pod", container: "api", init: false, root_uid: "root", root_kind: "Deployment", root_name: "checkout", observed_at: change.at, type: "termination", time_source: "finished_at", restart_count: 1 }; +const crashPage: KubernetesCrashPage = { items: [event], since: change.at, until: change.at, retention_from: change.at, retention_basis: "capture", started_at: change.at, gaps: [], capture_gaps: [], coverage: { Pod: { from: change.at, to: change.at } }, limitations: [], capture: { mode: "continuous", behind: false }, sync }; + +describe("Focused history continuation", () => { + it("follows an empty intermediate page, deduplicates stable IDs and retains every gap and failure", async () => { + const gap = { from: "2026-10-14T11:00:00Z", to: change.at }; + const fetch = vi.fn<(cursor?: string) => Promise>() + .mockResolvedValueOnce({ items: [change], next: "a+/=", sync, gaps: [gap] }) + .mockResolvedValueOnce({ items: [], next: "b+/=", sync, partial_failures: [{ class: "storage_unavailable" }] }) + .mockResolvedValueOnce({ items: [change, { ...change, id: "older", at: "2026-10-14T11:59:00Z" }], sync, gaps: [gap] }); + const result = await focusedChanges(fetch, new AbortController().signal); + expect(fetch.mock.calls.map(([cursor]) => cursor)).toEqual([undefined, "a+/=", "b+/="]); + expect(result.items.map(item => item.id)).toEqual(["older", "image"]); + expect(result.gaps).toEqual([gap]); + expect(result.partial_failures).toEqual([{ class: "storage_unavailable" }]); + expect(result.incomplete_range).toBe(false); + expect("pages" in result).toBe(false); + }); + + it("bounds empty continuations by requests and repeated opaque cursors", async () => { + const endless = vi.fn(async (cursor?: string) => ({ items: [], next: `${cursor ?? ""}x`, sync })); + expect((await focusedChanges(endless, new AbortController().signal)).incomplete_range).toBe(true); + expect(endless).toHaveBeenCalledTimes(focusedHistoryRequests); + const repeated = vi.fn(async () => ({ items: [change], next: "repeat", sync })); + const result = await focusedChanges(repeated, new AbortController().signal); + expect(repeated).toHaveBeenCalledTimes(2); + expect(result.items).toHaveLength(1); + expect(result.incomplete_range).toBe(true); + }); + + it("bounds rendered evidence even if a transport returns too many records", async () => { + const fetch = vi.fn(async () => ({ items: Array.from({ length: 501 }, (_, i) => ({ ...change, id: String(i) })), sync })); + const result = await focusedChanges(fetch, new AbortController().signal); + expect(result.items).toHaveLength(500); + expect(result.incomplete_range).toBe(true); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("retains admitted evidence on continuation failure but never hides authority loss", async () => { + const transport = vi.fn<(cursor?: string) => Promise>() + .mockResolvedValueOnce({ items: [change], next: "next", sync }) + .mockRejectedValueOnce(new ApiError(503, "unavailable")); + expect(await focusedChanges(transport, new AbortController().signal)).toMatchObject({ items: [change], incomplete_range: true, continuation_failed: true }); + for (const status of [401, 403, 404]) { + const denied = vi.fn<(cursor?: string) => Promise>() + .mockResolvedValueOnce({ items: [change], next: "next", sync }) + .mockRejectedValueOnce(new ApiError(status, "denied")); + await expect(focusedChanges(denied, new AbortController().signal)).rejects.toMatchObject({ status }); + } + }); + + it("scrubs late ignored-abort responses and does not continue the obsolete chain", async () => { + const controller = new AbortController(); + let finish!: (value: KubernetesChangesPage) => void; + const fetch = vi.fn(() => new Promise(resolve => { finish = resolve; })); + const result = focusedChanges(fetch, controller.signal); + controller.abort(); + finish({ items: [change], next: "next", sync }); + await expect(result).rejects.toMatchObject({ name: "AbortError" }); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("merges crashes through empty pages with conservative capture coverage", async () => { + const gap = { from: change.at, to: "2026-10-14T12:01:00Z" }; + const fetch = vi.fn<(cursor?: string) => Promise>() + .mockResolvedValueOnce({ ...crashPage, next: "a", capture_gaps: [gap] }) + .mockResolvedValueOnce({ ...crashPage, items: [], next: "b", coverage: {}, limitations: ["outside_retention"] }) + .mockResolvedValueOnce({ ...crashPage, items: [event, { ...event, id: "second" }], capture: { mode: "continuous", behind: true } }); + const result = await focusedCrashes(fetch, new AbortController().signal); + expect(result.items.map(item => item.id)).toEqual(["crash", "second"]); + expect(result.coverage).toEqual({}); + expect(result.capture_gaps).toEqual([gap]); + expect(result.limitations).toEqual(["outside_retention"]); + expect(result.capture.behind).toBe(true); + expect(result.crash_count).toBe(2); + expect(result.incomplete_range).toBe(false); + }); + + it("keeps a later-page capture failure rather than presenting first-page coverage as complete", async () => { + const fetch = vi.fn<(cursor?: string) => Promise>() + .mockResolvedValueOnce({ ...crashPage, next: "next" }) + .mockResolvedValueOnce({ ...crashPage, capture: { mode: "continuous", behind: false, evidence_error: "storage_unavailable" } }); + const result = await focusedCrashes(fetch, new AbortController().signal); + expect(result.incomplete_range).toBe(true); + expect(result.capture.evidence_error).toBe("storage_unavailable"); + expect(result.items).toHaveLength(1); + }); +}); diff --git a/ui/src/lib/kubernetesFocusedHistory.ts b/ui/src/lib/kubernetesFocusedHistory.ts new file mode 100644 index 00000000..07794c4a --- /dev/null +++ b/ui/src/lib/kubernetesFocusedHistory.ts @@ -0,0 +1,89 @@ +import type { KubernetesChange, KubernetesChangesPage, KubernetesCrashEvent, KubernetesCrashPage, KubernetesSync } from "./api"; +import { kubernetesAccessLost } from "./kubernetesContinuity"; + +// Match the existing bounded history scan: five requests, never unlimited history. +export const focusedHistoryRequests = 5; +export type FocusedCoverage = { incomplete_range: boolean; continuation_failed?: boolean }; + +type Page = { items: T[]; next?: string; sync: KubernetesSync }; +function unique(values: T[]): T[] { + return [...new Map(values.map(value => [JSON.stringify(value), value])).values()]; +} + +async function collect>( + fetch: (cursor?: string) => Promise

, signal: AbortSignal, itemBudget: number, +) { + const pages: P[] = []; + const items = new Map(); + const seen = new Set(); + let next: string | undefined; + let limited = false; + let failed = false; + for (let request = 0; request < focusedHistoryRequests; request++) { + signal.throwIfAborted(); + let page: P; + try { page = await fetch(next); } + catch (error) { + signal.throwIfAborted(); + if (!pages.length || kubernetesAccessLost(error)) throw error; + failed = true; + break; + } + signal.throwIfAborted(); + pages.push(page); + for (const item of page.items) { + if (items.has(item.id)) continue; + if (items.size === itemBudget) { limited = true; break; } + items.set(item.id, item); + } + next = page.next || undefined; + if (!next || limited) break; + if (seen.has(next)) { limited = true; break; } + seen.add(next); + } + return { + pages, items: [...items.values()].sort((a, b) => Date.parse(a.at) - Date.parse(b.at) || a.id.localeCompare(b.id)), + next, incomplete_range: Boolean(next || limited || failed), continuation_failed: failed, + }; +} + +function syncOf(pages: Array<{ sync: KubernetesSync }>): KubernetesSync { + const incomplete = pages.find(page => page.sync.state !== "ready" || page.sync.partial); + return { ...(incomplete ?? pages[0]).sync, age_s: Math.max(...pages.map(page => page.sync.age_s)), + partial: pages.some(page => page.sync.partial) }; +} + +export async function focusedChanges(fetch: (cursor?: string) => Promise, signal: AbortSignal): Promise { + const { pages, ...result } = await collect(fetch, signal, 500); + return { ...pages[0], ...result, sync: syncOf(pages), + truncated: result.incomplete_range || pages.some(page => page.truncated), + gaps: unique(pages.flatMap(page => page.gaps ?? [])), + partial_failures: unique(pages.flatMap(page => page.partial_failures ?? [])) }; +} + +export async function focusedCrashes(fetch: (cursor?: string) => Promise, signal: AbortSignal): Promise { + const { pages, ...result } = await collect(fetch, signal, 250); + const first = pages[0]; + const captureFailure = pages.find(page => page.capture.evidence_error); + // Coverage is an intersection: later pages cannot erase an earlier gap. + const coverage = { ...first.coverage }; + for (const page of pages) { + for (const kind of Object.keys(coverage)) { + const a = coverage[kind], b = page.coverage[kind]; + if (!b || !Number.isFinite(Date.parse(a.from)) || !Number.isFinite(Date.parse(b.from)) + || !Number.isFinite(Date.parse(a.to)) || !Number.isFinite(Date.parse(b.to))) { delete coverage[kind]; continue; } + const from = Math.max(Date.parse(a.from), Date.parse(b.from)); + const to = Math.min(Date.parse(a.to), Date.parse(b.to)); + if (from > to) { delete coverage[kind]; continue; } + coverage[kind] = { from: new Date(from).toISOString(), to: new Date(to).toISOString() }; + } + } + return { ...first, ...result, sync: syncOf(pages), coverage, + incomplete_range: result.incomplete_range || Boolean(captureFailure), + crash_count: result.items.filter(item => item.type === "termination").length, + gaps: unique(pages.flatMap(page => page.gaps ?? [])), + capture_gaps: unique(pages.flatMap(page => page.capture_gaps ?? [])), + limitations: unique(pages.flatMap(page => page.limitations ?? [])), + capture: { ...(captureFailure ?? first).capture, behind: pages.some(page => page.capture.behind), + mode: pages.every(page => page.capture.mode === first.capture.mode) ? first.capture.mode : "partial" } }; +} diff --git a/ui/src/lib/useKubernetesBreakLogs.ts b/ui/src/lib/useKubernetesBreakLogs.ts index e02e1f98..e501e38b 100644 --- a/ui/src/lib/useKubernetesBreakLogs.ts +++ b/ui/src/lib/useKubernetesBreakLogs.ts @@ -1,8 +1,9 @@ -import { createContext, useCallback, useContext, useEffect, useState } from "react"; +import { createContext, useCallback, useContext, useEffect, useLayoutEffect, useState } from "react"; import { useInfiniteQuery, useQueryClient } from "@tanstack/react-query"; import { useNavigate, useSearchParams } from "react-router-dom"; -import { ApiError, kubernetesAllContainers, type KubernetesBreakLogQuery, type KubernetesCrashEvent, type KubernetesIssue, type KubernetesResource } from "./api"; +import { kubernetesAllContainers, type KubernetesBreakLogQuery, type KubernetesCrashEvent, type KubernetesIssue, type KubernetesResource } from "./api"; import { useKubernetesCluster } from "./useKubernetesCluster"; +import { kubernetesAccessLost } from "./kubernetesContinuity"; export const maximumLogWindow = 2 * 60 * 60_000; export const CrashLogNavigationContext = createContext<{ current: ((event: KubernetesCrashEvent) => boolean) | null } | null>(null); @@ -75,7 +76,14 @@ export function useBreakLogsState(issue: KubernetesIssue | undefined, onSelectRe document.removeEventListener("visibilitychange", visibilityChanged); }; }, [validIssue, query.isError, firstPageFetchedAt, settleAt, queryClient, pollingQuery]); - const denied = query.error instanceof ApiError && [401, 403, 404].includes(query.error.status); + const denied = kubernetesAccessLost(query.error); + const protectedKey = JSON.stringify(queryKey); + useLayoutEffect(() => { + if (!denied) return; + const queryKey = JSON.parse(protectedKey) as readonly unknown[]; + void queryClient.cancelQueries({ queryKey, exact: true }); + queryClient.getQueryCache().find({ queryKey, exact: true })?.setState({ data: undefined }); + }, [denied, protectedKey, queryClient]); const page = denied ? undefined : query.data?.pages[0]; const lines = denied ? [] : query.data?.pages.flatMap((item) => item.lines ?? []) ?? []; const setWindow = (from: number, to: number) => { diff --git a/ui/src/lib/useKubernetesCluster.tsx b/ui/src/lib/useKubernetesCluster.tsx index fc86227d..f313bc37 100644 --- a/ui/src/lib/useKubernetesCluster.tsx +++ b/ui/src/lib/useKubernetesCluster.tsx @@ -3,6 +3,7 @@ import { useQuery, useQueryClient, type UseQueryOptions } from "@tanstack/react- import { api, createKubernetesApi, type KubernetesClusters, type KubernetesCrashQuery } from "./api"; import { useCursorPagination } from "./pagination"; import { hasKubernetesPublication, kubernetesAccessLost, retainKubernetesPublication } from "./kubernetesContinuity"; +import { focusedCrashes } from "./kubernetesFocusedHistory"; export function kubernetesKey(cluster: string | undefined, ...parts: readonly unknown[]) { return ["kubernetes", cluster ?? "_", ...parts] as const; @@ -36,7 +37,7 @@ export function useKubernetesCluster(): ClusterContext { return useContext(KubernetesClusterContext) ?? singleContext; } -export function useKubernetesQuery(options: UseQueryOptions) { +export function useKubernetesQuery(options: UseQueryOptions, firstReadyAt?: string) { const { key } = useKubernetesCluster(); const cache = useQueryClient(); const query = useQuery({ @@ -54,7 +55,7 @@ export function useKubernetesQuery(options: UseQueryOptions

-
; diff --git a/ui/src/pages/KubernetesBreakTimeline.test.tsx b/ui/src/pages/KubernetesBreakTimeline.test.tsx index 16df96ea..1565d2e8 100644 --- a/ui/src/pages/KubernetesBreakTimeline.test.tsx +++ b/ui/src/pages/KubernetesBreakTimeline.test.tsx @@ -29,12 +29,18 @@ function timelinePage(items: KubernetesTimelineEntry[] = [], overrides: Partial< } const changeEntry = (value = change): KubernetesTimelineEntry => ({ type: "change", id: value.id, at: value.at, change: value }); function persistedRootPage(value = issue, sync: KubernetesIssuePage["sync"] = { state: "ready", age_s: 0, partial: false }): KubernetesIssuePage { - return { items: [], totals: {}, truncated: false, sync, root_context: { root: value.root, occurrence: value, retained: true } }; + return { items: [], totals: {}, truncated: false, sync, selected_occurrence: value, selection_status: "resolved", root_context: { root: value.root, occurrence: value, retained: true } }; } function rootIssueResponses(page: KubernetesIssuePage) { return async (options: Parameters[0]) => options?.context === "persisted" ? persistedRootPage(page.items[0] ?? issue, page.sync) : page; } function Location() { const location = useLocation(); return {location.pathname}{location.search}; } +function ClickedBreak({ value }: { value: KubernetesIssue }) { + const location = useLocation(); + return new URLSearchParams(location.search).get("view") === "timeline" + ? + : ; +} function mount(element: React.ReactNode, cluster = "east") { const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); return { client, ...render( kubernetesKey(cluster, ...parts) }}>{element}) }; @@ -590,6 +596,31 @@ describe("Break timeline", () => { expect(screen.getByRole("region", { name: "Resource timeline chart" }).getAttribute("data-until")).toBe("2026-10-16T12:25:00.000Z"); }); + it.each(["resolved", "not_found"] as const)("preserves a clicked nanosecond break center and only enables logs after server resolution: %s", async (status) => { + const precise = { ...issue, broke_at: "2026-10-14T12:04:00.123456789Z" }; + const lookup = vi.spyOn(api, "kubernetesIssues").mockImplementation(async options => { + if (options?.context !== "persisted") return { items: [precise], totals: {}, truncated: false, sync: journalPage.sync }; + const resolved = options.center === precise.broke_at && status === "resolved"; + return { ...persistedRootPage(issue), selection_status: resolved ? "resolved" : "not_found", selected_occurrence: resolved ? precise : undefined }; + }); + vi.spyOn(api, "kubernetesCrashes").mockResolvedValue(coveredJournalPage); + vi.spyOn(api, "kubernetesChanges").mockResolvedValue({ items: [], sync: journalPage.sync }); + const logs = vi.spyOn(api, "kubernetesBreakLogs").mockResolvedValue({ window: { from: "2026-10-14T11:59:00Z", to: "2026-10-14T12:09:00Z" }, pods: [], lines: [], markers: [], counts: { all: 0, errors: 0, warnings: 0 }, rate: [], patterns: [], sources: [], coverage: { state: "complete", gaps: [] } }); + mount(); + fireEvent.click(screen.getByRole("button", { name: "Timeline for checkout" })); + await waitFor(() => expect(lookup.mock.calls.find(([options]) => options?.context === "persisted")?.[0]).toMatchObject({ center: precise.broke_at, occurrence_id: undefined })); + const location = screen.getByLabelText("Location").textContent!; + expect(new URLSearchParams(location.slice(location.indexOf("?"))).get("center")).toBe(precise.broke_at); + if (status === "resolved") { + await waitFor(() => expect(logs).toHaveBeenCalled()); + expect(logs.mock.calls[0][0]).toMatchObject({ since: "2026-10-14T11:59:00.123Z", until: "2026-10-14T12:06:00.123Z" }); + expect(screen.queryByText("Log time range is unavailable.")).toBeNull(); + } else { + await screen.findByText(/Log time range is unavailable/); + expect(logs).not.toHaveBeenCalled(); + } + }); + it.each([["pod.config_error", "Container config error"], ["pod.evicted", "Evicted"], ["pod.unschedulable", "Unschedulable"]])("names rule %s and singular affected Pods", (rule, label) => { mount(); expect(screen.getByLabelText(label)).toBeTruthy(); @@ -743,15 +774,13 @@ describe("Break timeline", () => { expect(screen.queryByRole("button", { name: /unrelated-0:/ })).toBeNull(); expect(graph).not.toHaveBeenCalled(); expect(related).not.toHaveBeenCalled(); - expect(lookup).toHaveBeenCalledTimes(1); + expect(lookup).toHaveBeenCalledTimes(2); expect(lookup.mock.calls[0][0]).toMatchObject({ namespace: "shop", root_uid: "checkout-uid", limit: 100, cursor: undefined }); - fireEvent.click(screen.getByRole("button", { name: "Next page" })); expect(await screen.findByRole("button", { name: /ConfigMap settings-23:.* at / })).toBeTruthy(); expect(lookup.mock.calls[1][0]).toMatchObject({ root_uid: "checkout-uid", cursor: "opaque+/=next" }); expect(lookup).toHaveBeenCalledTimes(2); - expect(screen.queryByRole("button", { name: /ConfigMap settings-0:.* at / })).toBeNull(); - expect(screen.getByRole("button", { name: "Next page" }).hasAttribute("disabled")).toBe(true); - fireEvent.click(screen.getByRole("button", { name: "Previous page" })); + expect(screen.queryByRole("button", { name: "Next page" })).toBeNull(); + expect(screen.queryByRole("button", { name: "Previous page" })).toBeNull(); expect(await screen.findByRole("button", { name: /ConfigMap settings-0:.* at / })).toBeTruthy(); const query = new URLSearchParams(screen.getByLabelText("Location").textContent!.split("?")[1]); expect(query.get("root_uid")).toBe(issue.root.uid); @@ -759,7 +788,7 @@ describe("Break timeline", () => { expect(query.get("center")).toBe(issue.broke_at); }); - it("continues opaque root cursors beyond five pages without eager reads or graph-based filtering", async () => { + it("bounds internal root continuation to five pages with loaded markers and explicit incomplete coverage", async () => { window.history.replaceState(null, "", `/agent/kubernetes?view=timeline&root_uid=checkout-uid&namespace=shop¢er=${issue.broke_at}`); const sync = { state: "ready", age_s: 0, partial: false }; vi.spyOn(api, "kubernetesIssues").mockImplementation(rootIssueResponses({ items: [issue], totals: {}, truncated: false, sync })); @@ -772,20 +801,12 @@ describe("Break timeline", () => { }); mount(); await screen.findByRole("button", { name: /Deployment linked-0:.* at / }); - expect(lookup).toHaveBeenCalledTimes(1); - for (let index = 1; index < cursors.length; index++) { - fireEvent.click(screen.getByRole("button", { name: "Next page" })); - await screen.findByRole("button", { name: new RegExp(`Deployment linked-${index}:.* at `) }); - expect(screen.queryByRole("button", { name: new RegExp(`Deployment linked-${index - 1}:.* at `) })).toBeNull(); - expect(lookup).toHaveBeenCalledTimes(index + 1); - } - expect(lookup).toHaveBeenCalledTimes(7); - expect(lookup.mock.calls.map(([options]) => options?.cursor)).toEqual(cursors); + expect(lookup).toHaveBeenCalledTimes(5); + for (let index = 1; index < 5; index++) expect(screen.getByRole("button", { name: new RegExp(`Deployment linked-${index}:.* at `) })).toBeTruthy(); + expect(lookup.mock.calls.map(([options]) => options?.cursor)).toEqual(cursors.slice(0, 5)); expect(lookup.mock.calls.every(([options]) => options?.root_uid === "checkout-uid" && options.namespace === "shop" && options.limit === 100)).toBe(true); - expect(screen.getByRole("button", { name: "Next page" }).hasAttribute("disabled")).toBe(true); - expect(screen.queryByText(/Related change scan is incomplete after 5 pages/)).toBeNull(); - fireEvent.click(screen.getByRole("button", { name: "Previous page" })); - await screen.findByRole("button", { name: /Deployment linked-5:.* at / }); + expect(screen.queryByRole("button", { name: "Next page" })).toBeNull(); + expect(screen.getByText(/Change evidence is incomplete for this range/)).toBeTruthy(); }); it("delegates root and URL kind filtering to the server and hides namespace and kind selectors in break detail", async () => { @@ -798,7 +819,6 @@ describe("Break timeline", () => { const { client } = mount(); expect(await screen.findByRole("button", { name: /Deployment checkout:.* at / })).toBeTruthy(); expect(lookup.mock.calls.at(-1)?.[0]).toMatchObject({ root_uid: "checkout-uid", namespace: "shop", kind: "Deployment", limit: 100 }); - fireEvent.click(screen.getByRole("button", { name: "Next page" })); await screen.findByRole("button", { name: /Deployment older-checkout:.* at / }); expect(client.getQueryCache().findAll().filter((query) => query.queryKey.includes("kubernetes-changes")).every((query) => query.queryKey[0] === "kubernetes" && query.queryKey[1] === "east")).toBe(true); expect(screen.queryByRole("combobox", { name: "Timeline kind" })).toBeNull(); @@ -1018,7 +1038,7 @@ describe("Break timeline", () => { expect(screen.getByText("Change capture does not cover the whole range")).toBeTruthy(); }); - it("does not follow a repeated root change cursor automatically or enable another next page", async () => { + it("stops repeated internal root cursors and reports incomplete evidence without a pager", async () => { window.history.replaceState(null, "", `/agent/kubernetes?view=timeline&root_uid=checkout-uid&namespace=shop¢er=${issue.broke_at}`); const sync = { state: "ready", age_s: 0, partial: false }; vi.spyOn(api, "kubernetesIssues").mockImplementation(rootIssueResponses({ items: [issue], totals: {}, truncated: false, sync })); @@ -1026,10 +1046,8 @@ describe("Break timeline", () => { vi.spyOn(api, "kubernetesCrashes").mockResolvedValue(coveredJournalPage); const lookup = vi.spyOn(api, "kubernetesChanges").mockResolvedValue({ items: [], next: "repeat", sync }); mount(); - await waitFor(() => expect(screen.getByRole("button", { name: "Next page" }).hasAttribute("disabled")).toBe(false)); - expect(lookup).toHaveBeenCalledOnce(); - fireEvent.click(screen.getByRole("button", { name: "Next page" })); - await waitFor(() => expect(screen.getByRole("button", { name: "Next page" }).hasAttribute("disabled")).toBe(true)); + await screen.findByText(/Change evidence is incomplete for this range/); + expect(screen.queryByRole("button", { name: "Next page" })).toBeNull(); expect(lookup).toHaveBeenCalledTimes(2); expect(screen.queryByText("Related change history is unavailable.")).toBeNull(); }); diff --git a/ui/src/pages/KubernetesBreakTimeline.tsx b/ui/src/pages/KubernetesBreakTimeline.tsx index b9d477e7..6f1f2a2b 100644 --- a/ui/src/pages/KubernetesBreakTimeline.tsx +++ b/ui/src/pages/KubernetesBreakTimeline.tsx @@ -228,6 +228,7 @@ export function CrashJournal({ journal, reasons = [] }: { journal: ReturnType Date.parse(page.since) || Date.parse(podCoverage.to) < Date.parse(page.until) || page.sync.state !== "ready" || page.sync.partial || page.capture.mode !== "continuous" || page.capture.behind || page.gaps?.length || page.capture_gaps?.length || windowLimited || ambiguousEvidence); return

Captured crash evidence

{page && {counted(page.crash_count ?? page.items.filter((event) => event.type === "termination").length, "crash")}}
+ {journal.focused && page && ("incomplete_range" in page && page.incomplete_range || page.next) &&

Crash evidence is incomplete for this range. Narrow the timeline window to inspect more evidence.

} {journal.isPending &&

Loading captured crash evidence...

} {journal.isError &&

Crash journal unavailable.

} {page && <> @@ -236,14 +237,14 @@ export function CrashJournal({ journal, reasons = [] }: { journal: ReturnTypeWindow: {journalTime(page.since)} – {journalTime(page.until)}

Capture started: {journalTime(page.started_at)}

- {!page.items.length &&

No captured observations on this page.

} -
    {page.items.map((event) =>
  • + {!page.items.length &&

    {journal.focused ? "No captured observations in the loaded range." : "No captured observations on this page."}

    } +
      {page.items.map((event) =>
    • {event.type === "termination" ? event.reason || "Terminated" : event.waiting_reason || "Status observation"}{" - "}{event.namespace}/{event.pod}{" / "}{event.init ? "Init " : ""}{event.container}{" / "}Observed: {event.time_source === "finished_at" && (Date.parse(event.at) < Date.parse(page.since) || Date.parse(event.at) > Date.parse(page.until)) && Reported finish outside observed window}

      {crashObservationDetail(event)}

      Root UID: {event.root_uid}; Pod UID: {event.pod_uid}

      Event time: ({event.time_source === "finished_at" ? "reported finish time" : "observation time; finish time unknown"})

      Observed:

      {event.ancestry_partial &&

      Capture-time ancestry is incomplete.

      }

      Observation ID: {event.id}

      • )}
    } - + {!journal.focused && }
; } @@ -276,6 +277,9 @@ export function BreakRow({ issue, resource, onSelectResource, collapsed = false, else query.delete("namespace"); if (issue.broke_at) query.set("center", issue.broke_at); else query.delete("center"); + if (issue.occurrence_id) query.set("occurrence_id", issue.occurrence_id); + else query.delete("occurrence_id"); + query.delete("crash_id"); navigate(`/agent/kubernetes?${query}`); }; const investigate = () => { diff --git a/ui/src/pages/KubernetesExplorerFlow.test.tsx b/ui/src/pages/KubernetesExplorerFlow.test.tsx index 1c863515..25808af5 100644 --- a/ui/src/pages/KubernetesExplorerFlow.test.tsx +++ b/ui/src/pages/KubernetesExplorerFlow.test.tsx @@ -37,7 +37,7 @@ function timelinePage(items: KubernetesTimelineEntry[], overrides: Partial[0]) => options?.context === "persisted" ? persistedRootPage(page.items[0] ?? issue, page.sync) : page; @@ -393,6 +393,113 @@ describe("Timeline range and filters", () => { }); describe("Persisted root identity independent of historical windows", () => { + it("uses the selected crash's verified observation time and keeps that anchor across range changes", async () => { + const event: KubernetesCrashEvent = { id: "selected-crash", sequence: 1, cluster: "east", namespace: "shop", pod: "checkout-a", pod_uid: "pod-a", + container: "api", init: false, root_uid: "checkout-uid", root_kind: "Deployment", root_name: "checkout", + type: "termination", at: "2026-10-14T12:02:00Z", observed_at: "2026-10-14T12:02:00Z", time_source: "observed_at", restart_count: 2 }; + window.history.replaceState(null, "", `/agent/kubernetes?view=timeline&root_uid=checkout-uid&namespace=shop&crash_id=selected-crash¢er=${event.at}`); + vi.spyOn(api, "kubernetesIssues").mockResolvedValue(persistedRootPage()); + vi.spyOn(api, "kubernetesChanges").mockResolvedValue({ items: [], sync }); + vi.spyOn(api, "kubernetesCrashes").mockResolvedValueOnce({ ...crashPage, items: [event] }).mockResolvedValue({ ...crashPage, items: [] }); + const logs = vi.spyOn(api, "kubernetesBreakLogs").mockResolvedValue(logPage()); + mount(); + await screen.findByRole("slider", { name: "Log window start" }); + expect(logs.mock.calls[0][0].since).toBe("2026-10-14T11:57:00.000Z"); + expect(screen.getByText("Observation time; finish time unknown")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "6h" })); + await waitFor(() => expect(api.kubernetesCrashes).toHaveBeenCalledTimes(2)); + expect(screen.getByRole("slider", { name: "Log window start" })).toBeTruthy(); + expect(logs.mock.calls.every(([options]) => options.since === "2026-10-14T11:57:00.000Z")).toBe(true); + }); + + it("scrubs protected logs and bracket metadata on a typed credential failure, then re-proves context before retrying", async () => { + window.history.replaceState(null, "", `/agent/kubernetes?view=timeline&root_uid=checkout-uid&namespace=shop¢er=${issue.broke_at}`); + const issues = vi.spyOn(api, "kubernetesIssues").mockImplementation(rootIssueResponses({ items: [issue], totals: {}, truncated: false, sync })); + vi.spyOn(api, "kubernetesChanges").mockResolvedValue({ items: [], sync }); + vi.spyOn(api, "kubernetesCrashes").mockResolvedValue(crashPage); + const logs = vi.spyOn(api, "kubernetesBreakLogs").mockResolvedValue(logPage()); + const { client } = mount(); + await screen.findByText("upstream timeout"); + const logQuery = client.getQueryCache().findAll().find(query => query.queryKey.includes("kubernetes-break-logs") && query.isActive())!; + logs.mockRejectedValue(new ApiError(502, "credential unavailable", { code: "credential_unavailable" })); + await act(async () => { await client.invalidateQueries({ queryKey: logQuery.queryKey, exact: true }); }); + await screen.findByRole("button", { name: "Retry root context" }); + expect(screen.queryByText("upstream timeout")).toBeNull(); + expect(screen.queryByRole("slider", { name: "Log window start" })).toBeNull(); + expect(client.getQueryData(logQuery.queryKey)).toBeUndefined(); + const proofs = issues.mock.calls.filter(([options]) => options?.context === "persisted").length; + logs.mockResolvedValue(logPage()); + fireEvent.click(screen.getByRole("button", { name: "Retry root context" })); + await screen.findByText("upstream timeout"); + expect(issues.mock.calls.filter(([options]) => options?.context === "persisted")).toHaveLength(proofs + 1); + }); + + it("pins the clicked resolved occurrence rather than a later recurrence as bounded history arrives", async () => { + const selected = { ...issue, occurrence_id: "selected-break", state: "resolved" as const, broken: false }; + const recurrence = { ...issue, occurrence_id: "later-break", broke_at: "2026-10-14T12:05:00Z" }; + window.history.replaceState(null, "", `/agent/kubernetes?view=timeline&root_uid=checkout-uid&namespace=shop&occurrence_id=selected-break¢er=${issue.broke_at}`); + const context = { ...persistedRootPage(recurrence), selected_occurrence: selected }; + const issues = vi.spyOn(api, "kubernetesIssues").mockImplementation(async options => options?.context === "persisted" ? context : { items: [recurrence], totals: {}, truncated: false, sync }); + const last = deferred<{ items: typeof change[]; sync: typeof sync }>(); + const changes = vi.spyOn(api, "kubernetesChanges").mockImplementation(async options => { + if (!options?.cursor) return { items: [change], next: "empty", sync }; + if (options.cursor === "empty") return { items: [], next: "last", sync }; + return last.promise; + }); + const crashes = vi.spyOn(api, "kubernetesCrashes").mockImplementation(async options => ({ + ...crashPage, items: options?.cursor === "empty" ? [] : crashPage.items, + next: !options?.cursor ? "empty" : options.cursor === "empty" ? "last" : undefined, + })); + const logs = vi.spyOn(api, "kubernetesBreakLogs").mockResolvedValue(logPage()); + mount(); + const panel = await screen.findByRole("region", { name: "Logs at break" }); + const bracket = await screen.findByRole("slider", { name: "Log window start" }); + const lane = screen.getByRole("region", { name: "Timeline lane checkout" }); + await waitFor(() => expect(changes).toHaveBeenCalledTimes(3)); + expect(issues.mock.calls.find(([options]) => options?.context === "persisted")?.[0]).toMatchObject({ occurrence_id: "selected-break", center: issue.broke_at }); + expect(logs.mock.calls[0][0].since).toBe("2026-10-14T11:59:00.000Z"); + fireEvent.click(within(panel).getByRole("button", { name: /^Errors/ })); + await act(async () => { last.resolve({ items: [change, { ...change, id: "second", name: "second" }], sync }); }); + await screen.findByRole("button", { name: /Deployment second:.* at / }); + expect(screen.getByRole("region", { name: "Logs at break" })).toBe(panel); + expect(screen.getByRole("slider", { name: "Log window start" })).toBe(bracket); + expect(screen.getByRole("region", { name: "Timeline lane checkout" })).toBe(lane); + expect(within(panel).getByRole("button", { name: /^Errors/ }).getAttribute("aria-pressed")).toBe("true"); + expect(changes.mock.calls.map(([options]) => options?.cursor)).toEqual([undefined, "empty", "last"]); + expect(crashes.mock.calls.map(([options]) => options?.cursor)).toEqual([undefined, "empty", "last"]); + expect(screen.queryByRole("navigation", { name: "Crash journal pages" })).toBeNull(); + expect(screen.queryByRole("button", { name: "Next page" })).toBeNull(); + expect(logs.mock.calls.every(([options]) => options.since === "2026-10-14T11:59:00.000Z")).toBe(true); + }); + + it.each(["not_found", "ambiguous", "partial", "missing-time"] as const)("keeps a truthful mounted log panel for %s without using URL center or another occurrence as a break", async status => { + window.history.replaceState(null, "", `/agent/kubernetes?view=timeline&root_uid=checkout-uid&namespace=shop&occurrence_id=selected-break¢er=${issue.broke_at}`); + const context: KubernetesIssuePage = { ...persistedRootPage(), selection_status: status === "missing-time" ? "resolved" : status, + selected_occurrence: status === "missing-time" ? { ...issue, occurrence_id: "selected-break", broke_at: undefined } : undefined }; + vi.spyOn(api, "kubernetesIssues").mockResolvedValue(context); + vi.spyOn(api, "kubernetesChanges").mockResolvedValue({ items: [], sync }); + vi.spyOn(api, "kubernetesCrashes").mockResolvedValue(crashPage); + const logs = vi.spyOn(api, "kubernetesBreakLogs"); + mount(); + const panel = await screen.findByRole("region", { name: "Logs at break" }); + expect(await within(panel).findByText("Log time range is unavailable for this observation.")).toBeTruthy(); + expect(within(panel).getByRole("button", { name: "Ask chat" }).hasAttribute("disabled")).toBe(true); + expect(screen.queryByRole("slider", { name: "Log window start" })).toBeNull(); + expect(logs).not.toHaveBeenCalled(); + }); + + it("rejects a mismatched selected occurrence instead of substituting root identity", async () => { + window.history.replaceState(null, "", `/agent/kubernetes?view=timeline&root_uid=checkout-uid&namespace=shop&occurrence_id=selected-break¢er=${issue.broke_at}`); + vi.spyOn(api, "kubernetesIssues").mockResolvedValue({ ...persistedRootPage(), selected_occurrence: { ...issue, occurrence_id: "different-break" } }); + const logs = vi.spyOn(api, "kubernetesBreakLogs"); + const changes = vi.spyOn(api, "kubernetesChanges"); + mount(); + await screen.findByRole("button", { name: "Retry root context" }); + expect(screen.queryByRole("region", { name: "Logs at break" })).toBeNull(); + expect(logs).not.toHaveBeenCalled(); + expect(changes).not.toHaveBeenCalled(); + }); + it("keeps persisted identity when the selected window predates capture, without inventing markers or totals", async () => { window.history.replaceState(null, "", "/agent/kubernetes?view=timeline&root_uid=checkout-uid&namespace=shop¢er=2020-01-01T00:00:00Z"); const contextPage = persistedRootPage({ ...issue, first_seen: "2020-01-01T00:00:00Z", changes: [{ change, relation: "same_object", lead_seconds: 1 }] }); @@ -411,8 +518,8 @@ describe("Persisted root identity independent of historical windows", () => { expect(related).not.toHaveBeenCalled(); const metadata = lookup.mock.calls.filter(([options]) => options?.context === "persisted"); expect(metadata).toHaveLength(1); - expect(metadata[0][0]).toEqual({ context: "persisted", root_uid: "checkout-uid", namespace: "shop" }); - const contextKey = kubernetesKey("east", "kubernetes-issues", "timeline-root-context", "checkout-uid", "shop"); + expect(metadata[0][0]).toMatchObject({ context: "persisted", root_uid: "checkout-uid", namespace: "shop", center: "2020-01-01T00:00:00Z" }); + const contextKey = kubernetesKey("east", "kubernetes-issues", "timeline-root-context", "checkout-uid", "shop", undefined, "2020-01-01T00:00:00Z"); expect(client.getQueryData(contextKey)).toMatchObject({ items: [], totals: {}, root_context: { retained: true, root: issue.root } }); expect(changes.mock.calls[0][0]).toMatchObject({ root_uid: "checkout-uid", namespace: "shop", since: "2019-12-31T23:30:00.000Z", until: "2020-01-01T00:30:00.000Z" }); expect(crashes.mock.calls[0][0]).toMatchObject({ root_uid: "checkout-uid", namespace: "shop" }); @@ -493,12 +600,11 @@ describe("Persisted root identity independent of historical windows", () => { }); const related = vi.spyOn(api, "kubernetesRelated"); const { client } = mount(); - await screen.findByRole("button", { name: /Deployment checkout:.* at / }); - fireEvent.click(screen.getByRole("button", { name: "Next page" })); + await screen.findAllByRole("button", { name: /Deployment checkout:.* at / }); await waitFor(() => expect(changes.mock.calls.at(-1)?.[0]?.cursor).toBe("older")); - await screen.findByRole("button", { name: /Deployment checkout:.* at / }); + await screen.findAllByRole("button", { name: /Deployment checkout:.* at / }); const selected = changes.mock.calls.at(-1)![0]; - const contextKey = kubernetesKey("east", "kubernetes-issues", "timeline-root-context", "checkout-uid", explicitNamespace ? "shop" : ""); + const contextKey = kubernetesKey("east", "kubernetes-issues", "timeline-root-context", "checkout-uid", explicitNamespace ? "shop" : "", undefined, issue.broke_at); const otherKey = kubernetesKey("east", "kubernetes-issues", "timeline-root-context", "other-root", "shop"); client.setQueryData(otherKey, { identity: "unaffected" }); const target = client.getQueryCache().findAll({ queryKey: kubernetesKey("east", `kubernetes-${route}`), predicate: query => query.isActive() }) @@ -521,10 +627,10 @@ describe("Persisted root identity independent of historical windows", () => { expect(screen.queryByText("Deployment shop/checkout")).toBeNull(); holdContext = false; await act(async () => { nextContext.resolve(persistedRootPage()); }); - expect(await screen.findByRole("button", { name: /Deployment checkout:.* at / })).toBeTruthy(); + expect(await screen.findAllByRole("button", { name: /Deployment checkout:.* at / })).toHaveLength(2); expect(screen.getByText("Deployment shop/checkout")).toBeTruthy(); expect(historyCounts().every((count, index) => count > beforeRetry[index])).toBe(true); - expect(contextCalls().at(-1)?.[0]).toEqual({ context: "persisted", root_uid: "checkout-uid", namespace: explicitNamespace ? "shop" : undefined }); + expect(contextCalls().at(-1)?.[0]).toMatchObject({ context: "persisted", root_uid: "checkout-uid", namespace: explicitNamespace ? "shop" : undefined, center: issue.broke_at }); expect(changes.mock.calls.at(-1)?.[0]).toEqual(selected); expect(crashes.mock.calls.every(([options]) => options?.namespace === "shop")).toBe(true); expect(client.getQueryData(contextKey)).toMatchObject({ items: [], totals: {}, root_context: { retained: true } }); @@ -552,7 +658,7 @@ describe("Persisted root identity independent of historical windows", () => { await waitFor(() => expect((screen.getByRole("button", { name: "Retry root context" }) as HTMLButtonElement).disabled).toBe(false)); expect(screen.queryByText("Deployment shop/checkout")).toBeNull(); expect(screen.queryByRole("region", { name: "Timeline lane checkout" })).toBeNull(); - expect(client.getQueryData(kubernetesKey("east", "kubernetes-issues", "timeline-root-context", "checkout-uid", "shop"))).toBeUndefined(); + expect(client.getQueryData(kubernetesKey("east", "kubernetes-issues", "timeline-root-context", "checkout-uid", "shop", undefined, issue.broke_at))).toBeUndefined(); } expect(issues.mock.calls.filter(([options]) => options?.context === "persisted")).toHaveLength(3); }); @@ -572,7 +678,7 @@ describe("Persisted root identity independent of historical windows", () => { vi.spyOn(api, "kubernetesBreakLogs").mockResolvedValue(logPage()); const { client } = mount(); await screen.findByRole("button", { name: /Deployment checkout:.* at / }); - const contextKey = kubernetesKey("east", "kubernetes-issues", "timeline-root-context", "checkout-uid", "shop"); + const contextKey = kubernetesKey("east", "kubernetes-issues", "timeline-root-context", "checkout-uid", "shop", undefined, issue.broke_at); const otherKey = kubernetesKey("east", "kubernetes-issues", "timeline-root-context", "different-uid", "shop"); client.setQueryData(otherKey, { retained: "other-root" }); const lateKey = kubernetesKey("east", "kubernetes-changes", "timeline-root", "checkout-uid", "shop", "late"); diff --git a/ui/src/pages/KubernetesExplorerViews.tsx b/ui/src/pages/KubernetesExplorerViews.tsx index 665a853e..90dbb604 100644 --- a/ui/src/pages/KubernetesExplorerViews.tsx +++ b/ui/src/pages/KubernetesExplorerViews.tsx @@ -19,6 +19,7 @@ import { ApiError, type KubernetesChange, type KubernetesChangesPage, + type KubernetesCrashEvent, type KubernetesHelmRelease, type KubernetesGraph, type KubernetesGraphNode, @@ -38,6 +39,7 @@ import { KubernetesWarnings } from "./KubernetesWarnings"; import { failureWarnings, issueWarnings } from "./kubernetesWarningItems"; import { KubernetesAgentTab } from "./KubernetesAgentTab"; import { KubernetesBreakLogs, KubernetesBreakLogsProvider } from "./KubernetesBreakLogs"; +import { focusedChanges, type FocusedCoverage } from "@/lib/kubernetesFocusedHistory"; const virtualViewportHeight = 440; const resourceRowHeight = 58; @@ -861,12 +863,14 @@ function TimelineView({ onSelectResource }: { onSelectResource: ResourceSelectio const { search } = useLocation(); const focus = new URLSearchParams(search); const rootUID = focus.get("root_uid"); - return rootUID ? : ; + return rootUID ? : ; } function RootTimelineView({ rootUID, focus, onSelectResource }: { rootUID: string; focus: URLSearchParams; onSelectResource: ResourceSelection }) { const { client: api, cluster, key } = useKubernetesCluster(); const center = focus.get("center"); + const occurrenceID = focus.get("occurrence_id") || undefined; + const crashID = focus.get("crash_id") || undefined; const centerTime = center ? Date.parse(center) : NaN; const centered = Number.isFinite(centerTime); const [minutes, setMinutes] = useState(60); @@ -879,10 +883,11 @@ function RootTimelineView({ rootUID, focus, onSelectResource }: { rootUID: strin const [authorityHeld, setAuthorityHeld] = useState(false); const [checkingContext, setCheckingContext] = useState(false); const [awaitingWindowProof, setAwaitingWindowProof] = useState(false); + const [logAuthorityError, setLogAuthorityError] = useState(null); const rootContext = useQuery({ - queryKey: ["kubernetes-issues", "timeline-root-context", rootUID, namespace], + queryKey: ["kubernetes-issues", "timeline-root-context", rootUID, namespace, occurrenceID, center], queryFn: async ({ signal }) => { - const page = await api.kubernetesIssues({ root_uid: rootUID, namespace: namespace || undefined, context: "persisted" }, signal); + const page = await api.kubernetesIssues({ root_uid: rootUID, namespace: namespace || undefined, context: "persisted", occurrence_id: occurrenceID, center: centered ? center! : undefined }, signal); const context = page.root_context; if (context && (context.retained !== true || !context.root.kind || !context.root.name || context.root.uid !== rootUID || context.occurrence.root.uid !== rootUID || namespace && (context.root.namespace !== namespace || context.occurrence.root.namespace !== namespace) @@ -890,6 +895,12 @@ function RootTimelineView({ rootUID, focus, onSelectResource }: { rootUID: strin || context.root.namespace !== context.occurrence.root.namespace)) { throw new ApiError(502, "Retained root context does not match this selection", { code: "root_context_mismatch" }); } + const selected = page.selected_occurrence; + if (selected && (!context || page.selection_status !== "resolved" || selected.root.uid !== rootUID + || selected.root.namespace !== context.root.namespace || selected.root.kind !== context.root.kind || selected.root.name !== context.root.name + || occurrenceID && selected.occurrence_id !== occurrenceID)) { + throw new ApiError(502, "Selected occurrence does not match this selection", { code: "root_context_mismatch" }); + } return page; }, retry: false, @@ -916,11 +927,10 @@ function RootTimelineView({ rootUID, focus, onSelectResource }: { rootUID: strin retry: false, refetchInterval: (query) => query.state.data?.sync.pending?.length ? 5000 : false, }); - const crashes = useKubernetesCrashes({ root_uid: rootUID, namespace: rootNamespace, since, until: new Date(Math.min(journalCeiling, Date.parse(until))).toISOString() }, windowReadsEnabled && !kubernetesAccessLost(breaks.error), true); + const crashes = useKubernetesCrashes({ root_uid: rootUID, namespace: rootNamespace, since, until: new Date(Math.min(journalCeiling, Date.parse(until))).toISOString() }, windowReadsEnabled && !kubernetesAccessLost(breaks.error), true, true); const historicalIssue = breaks.data?.items.find(item => item.root.uid === rootUID && item.root.namespace === context?.root.namespace); - const paging = useCursorPagination(`root-timeline:${rootUID}:${since}:${until}:${namespace}:${kind}`); - const changes = useQuery({ - queryKey: ["kubernetes-changes", "timeline-root", rootUID, rootNamespace, since, until, kind, paging.cursor], + const changes = useQuery({ + queryKey: ["kubernetes-changes", "timeline-root", rootUID, rootNamespace, since, until, kind], enabled: windowReadsEnabled && !kubernetesAccessLost(breaks.error) && !kubernetesAccessLost(crashes.error), retry: false, // Keep the previous range on screen while a new range or page loads, so @@ -930,13 +940,13 @@ function RootTimelineView({ rootUID, focus, onSelectResource }: { rootUID: strin const scope = key("kubernetes-changes", "timeline-root", rootUID, rootNamespace); return prior && JSON.stringify(prior.slice(0, scope.length)) === JSON.stringify(scope) && prior[scope.length + 2] === kind ? previous : undefined; }, - queryFn: ({ signal }) => api.kubernetesChanges({ root_uid: rootUID, since, until, namespace: rootNamespace, kind: kind || undefined, limit: 100, cursor: paging.cursor }, signal), + queryFn: ({ signal }) => focusedChanges(cursor => api.kubernetesChanges({ root_uid: rootUID, since, until, namespace: rootNamespace, kind: kind || undefined, limit: 100, cursor }, signal, "single"), signal), }); - const rootAuthorityLost = contextBlocked || kubernetesAccessLost(breaks.error) || kubernetesAccessLost(changes.error) || kubernetesAccessLost(crashes.error); + const rootAuthorityLost = contextBlocked || kubernetesAccessLost(breaks.error) || kubernetesAccessLost(changes.error) || kubernetesAccessLost(crashes.error) || kubernetesAccessLost(logAuthorityError); const fenced = useRef>([]); useLayoutEffect(() => { if (!rootAuthorityLost) { fenced.current = []; return; } - const errors = [rootContext.error, breaks.error, changes.error, crashes.error]; + const errors = [rootContext.error, breaks.error, changes.error, crashes.error, logAuthorityError]; if (errors.every((error, index) => error === fenced.current[index])) return; fenced.current = errors; setAuthorityHeld(true); @@ -954,8 +964,8 @@ function RootTimelineView({ rootUID, focus, onSelectResource }: { rootUID: strin void cache.cancelQueries(filters); cache.removeQueries({ ...filters, predicate: query => filters.predicate(query) && query.queryKey[3] !== "timeline-root-context" && !kubernetesAccessLost(query.state.error) }); - cache.getQueryCache().find({ queryKey: key("kubernetes-issues", "timeline-root-context", rootUID, namespace), exact: true })?.setState({ data: undefined }); - }, [rootAuthorityLost, rootContext.error, breaks.error, changes.error, crashes.error, cache, key, rootUID, namespace]); + cache.getQueryCache().find({ queryKey: key("kubernetes-issues", "timeline-root-context", rootUID, namespace, occurrenceID, center), exact: true })?.setState({ data: undefined }); + }, [rootAuthorityLost, rootContext.error, breaks.error, changes.error, crashes.error, logAuthorityError, cache, key, rootUID, namespace, occurrenceID, center]); useEffect(() => { if (!awaitingWindowProof) return; if (breaks.isError || changes.isError || crashes.isError) { setAwaitingWindowProof(false); return; } @@ -975,25 +985,38 @@ function RootTimelineView({ rootUID, focus, onSelectResource }: { rootUID: strin const fresh = result.data?.root_context; if (!recovering || result.isError || !fresh) return; const freshNamespace = fresh.root.namespace; + setLogAuthorityError(null); // A current date-free context proof precedes fresh selected-window reads. // Reset errors/data only; never change the user's range, kind or cursors. await Promise.all([ key("kubernetes-issues", "timeline-root", rootUID, freshNamespace ?? "", since, until), - key("kubernetes-changes", "timeline-root", rootUID, freshNamespace, since, until, kind, paging.cursor), - key("kubernetes-crashes", { root_uid: rootUID, namespace: freshNamespace, since, until: new Date(Math.min(journalCeiling, Date.parse(until))).toISOString() }, crashes.paging.cursor), + key("kubernetes-changes", "timeline-root", rootUID, freshNamespace, since, until, kind), + key("kubernetes-crashes", { root_uid: rootUID, namespace: freshNamespace, since, until: new Date(Math.min(journalCeiling, Date.parse(until))).toISOString() }, crashes.paging.cursor, "focused"), ].map(queryKey => cache.resetQueries({ queryKey, exact: true }))); setAwaitingWindowProof(true); } finally { setCheckingContext(false); } }; - const issue = !rootAuthorityLost && !authorityHeld && context ? { ...(historicalIssue ?? context.occurrence), root: context.root, changes: [] } : undefined; + const selectedOccurrence = rootContext.data?.selection_status === "resolved" ? rootContext.data.selected_occurrence : undefined; + const pinnedCrash = useRef(undefined); + if (rootAuthorityLost || authorityHeld || !context) pinnedCrash.current = undefined; + else if (crashID && !pinnedCrash.current) pinnedCrash.current = crashes.data?.items.find(event => event.id === crashID + && event.root_uid === rootUID && event.namespace === context.root.namespace && (!cluster || event.cluster === cluster) + && Number.isFinite(Date.parse(event.at)) && !isZeroTime(event.at)); + const selection: KubernetesIssue | undefined = crashID ? pinnedCrash.current && { + root: context!.root, rule: "container.termination", severity: "warning", count: 1, state: "observed", + broke_at: pinnedCrash.current.at, broke_at_source: pinnedCrash.current.time_source, + } : selectedOccurrence; + const verifiedTime = selection?.broke_at && !isZeroTime(selection.broke_at) + && Number.isFinite(Date.parse(selection.broke_at)) ? selection.broke_at : undefined; + const issue = !rootAuthorityLost && !authorityHeld && context ? { ...(selection ?? context.occurrence), broke_at: verifiedTime, root: context.root, changes: [] } : undefined; const data = issue && changes.data ? { ...changes.data, items: changes.data.items.filter((change) => (!cluster || change.cluster === cluster) && (!rootNamespace || change.namespace === rootNamespace || change.kind === "Node")) } : undefined; const gaps = [...(data?.gaps ?? []), ...(historicalIssue?.gaps ?? [])]; const failures = rootAuthorityLost ? [] : [...(rootContext.data?.partial_failures ?? []), ...(breaks.data?.partial_failures ?? []), ...(data?.partial_failures ?? [])]; - const warnings: Array<{ id: string; content: ReactNode }> = [...issueWarnings(historicalIssue ? { ...historicalIssue, gaps: [] } : undefined), ...failureWarnings(failures)]; + const warnings: Array<{ id: string; content: ReactNode }> = [...issueWarnings(selection ? { ...selection, gaps: [] } : undefined), ...failureWarnings(failures)]; if ([rootContext.data?.sync.state, breaks.data?.sync.state, data?.sync.state].includes("stale")) warnings.push({ id: "snapshot-stale", content: "Timeline evidence includes stale snapshots." }); if (gaps.length) warnings.push({ id: "history-gaps", content: "Change capture does not cover the whole range" }); const unfocusedQuery = new URLSearchParams(focus); - for (const name of ["root_uid", "center", "namespace", "kind"]) unfocusedQuery.delete(name); + for (const name of ["root_uid", "occurrence_id", "crash_id", "center", "namespace", "kind"]) unfocusedQuery.delete(name); if (namespace) unfocusedQuery.set("namespace", namespace); if (kind) unfocusedQuery.set("kind", kind); for (const name of ["logs_from", "logs_to", "logs_mode", "logs_pod", "logs_container"]) unfocusedQuery.delete(name); @@ -1002,7 +1025,8 @@ function RootTimelineView({ rootUID, focus, onSelectResource }: { rootUID: strin const noMarkers = issue && data && !data.items.length && !data.next && crashes.data && !crashes.data.items.length && !crashes.data.next && !breaks.isError && !changes.isError && !crashes.isError && !changes.isFetching && !crashes.isFetching && !rangeLoading && !incompleteKubernetesSnapshot(data) && !incompleteKubernetesSnapshot(crashes.data); - return
+ const selectedOffRange = verifiedTime && (Date.parse(verifiedTime) < Date.parse(since) || Date.parse(verifiedTime) > Date.parse(until)); + return
Filtered{issue ? `${issue.root.kind} ${issue.root.namespace ? `${issue.root.namespace}/` : ""}${issue.root.name}` : rootUID}{centered && }
@@ -1015,12 +1039,13 @@ function RootTimelineView({ rootUID, focus, onSelectResource }: { rootUID: strin {(breaks.isError || breaks.unpublished) &&

Root history unavailable.

} {changes.isError &&

Related change history is unavailable.

} {issue && <> - { const resource_id = resourceIdForKind(change.kind); if (resource_id) onSelectResource({ resource_id, kind: change.kind, namespace: change.namespace, name: change.name, uid: change.uid }, "timeline"); }} /> - + { const resource_id = resourceIdForKind(change.kind); if (resource_id) onSelectResource({ resource_id, kind: change.kind, namespace: change.namespace, name: change.name, uid: change.uid }, "timeline"); }} /> + {data?.incomplete_range &&

Change evidence is incomplete for this range. Narrow the timeline window to inspect more evidence.

} + {selectedOffRange &&

The selected break time is outside this timeline range. Its log window remains selected.

} {noMarkers &&

No captured markers in this range.

} }
- + {issue && warning.content)} />}
; } @@ -1071,11 +1096,14 @@ function UnfocusedTimelineView({ onSelectResource }: { onSelectResource: Resourc if (!resource_id) return; onSelectResource({ resource_id, kind: change.kind, namespace: change.namespace, name: change.name, uid: change.uid }, "timeline"); }; - const focusedLink = (uid: string, at: string, ns?: string) => { + const focusedLink = (uid: string, at: string, ns?: string, occurrenceID?: string, crashID?: string) => { const query = new URLSearchParams(focus); query.set("view", "timeline"); query.set("root_uid", uid); query.set("center", at); + if (occurrenceID) query.set("occurrence_id", occurrenceID); else query.delete("occurrence_id"); + if (crashID) query.set("crash_id", crashID); else query.delete("crash_id"); + for (const field of ["logs_from", "logs_to", "logs_mode", "logs_pod", "logs_container"]) query.delete(field); if (ns) query.set("namespace", ns); else query.delete("namespace"); return `?${query}`; }; @@ -1089,11 +1117,11 @@ function UnfocusedTimelineView({ onSelectResource }: { onSelectResource: Resourc if (entry.type === "break") { const issue = entry.issue; return { ...entry, kind: issue.root.kind, name: issue.root.name, namespace: issue.root.namespace, summary: issue.rule.replaceAll(/[._]/g, " "), - content: Open break details }; + content: Open break details }; } const crash = entry.crash; return { ...entry, kind: crash.root_kind || "Pod", name: crash.root_name || crash.pod, namespace: crash.namespace, summary: `${crash.container}: ${crash.reason || "Terminated"}, exit ${crash.exit_code}`, - content: Open crash details }; + content: Open crash details }; }); return
} title="Breaks and changes" trailing={{data && (data.items.length > 0 || !timeline.isError && !incompleteKubernetesSnapshot(data)) && {breaksOnly ? `${data.counts?.break ?? data.total} ${(data.counts?.break ?? data.total) === 1 ? "break" : "breaks"} in ${timelineRanges.find(([value]) => value === minutes)?.[1]}` : `${counted(data.total, "event")} and ${counted(data.crash_count, "crash")}`}}} /> diff --git a/ui/src/pages/KubernetesPage.test.tsx b/ui/src/pages/KubernetesPage.test.tsx index e9460d62..4b3d9bae 100644 --- a/ui/src/pages/KubernetesPage.test.tsx +++ b/ui/src/pages/KubernetesPage.test.tsx @@ -1,6 +1,6 @@ // @vitest-environment jsdom import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; -import { QueryClient, QueryClientProvider, QueryObserver } from "@tanstack/react-query"; +import { notifyManager, QueryClient, QueryClientProvider, QueryObserver } from "@tanstack/react-query"; import { BrowserRouter, useLocation } from "react-router-dom"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { api, ApiError, type KubernetesIssue } from "@/lib/api"; @@ -109,10 +109,182 @@ beforeEach(() => { afterEach(() => { cleanup(); vi.clearAllMocks(); vi.useRealTimers(); window.history.replaceState(null, "", "/"); }); describe("KubernetesPage", () => { + it.each(["Overview", "Issues", "Timeline"])("opens the pinned occurrence from %s without a cross-component render update", async (entrypoint) => { + const errors = vi.spyOn(console, "error"); + const sync = { state: "ready", age_s: 0, partial: false }; + const center = new Date(Date.now() - 300_000).toISOString().replace(/\.\d{3}Z$/, ".123456789Z"); + const selected: KubernetesIssue = { + root: { kind: "Deployment", namespace: "shop", name: "checkout", uid: "checkout-uid" }, + occurrence_id: "pinned-old", rule: "pod.crash_loop", severity: "critical", count: 1, broken: false, broke_at: center, + }; + const newer = { ...selected, occurrence_id: "newer", broke_at: new Date().toISOString() }; + vi.mocked(api.kubernetesIssues).mockImplementation(async (options) => ({ + items: [options?.context === "persisted" ? newer : selected], totals: { critical: 1 }, broken: { total: 1 }, sync, + ...(options?.context === "persisted" ? { selected_occurrence: selected, selection_status: "resolved" as const, root_context: { root: selected.root, occurrence: newer, retained: true } } : {}), + })); + vi.mocked(api.kubernetesTimeline).mockResolvedValue({ + items: [{ type: "break", id: selected.occurrence_id!, at: center, issue: selected }], + total: 1, crash_count: 0, counts: { change: 0, crash: 0, break: 1 }, since: center, until: new Date().toISOString(), gaps: [], sync, + }); + const logs = vi.spyOn(api, "kubernetesBreakLogs").mockImplementation(async (options) => ({ + window: { from: options.since, to: options.until }, pods: [], lines: [], markers: [], counts: { all: 0, errors: 0, warnings: 0 }, + rate: [], patterns: [], sources: [], coverage: { state: "complete", gaps: [], reason: "", tip: false }, + })); + try { + renderPage(); + await screen.findByRole("button", { name: "Timeline for checkout" }); + if (entrypoint !== "Overview") { + await browserClick(screen.getByRole("tab", { name: entrypoint, exact: true })); + } + await browserClick(entrypoint === "Timeline" + ? await screen.findByRole("link", { name: "Open break details", exact: true }) + : await screen.findByRole("button", { name: "Timeline for checkout" })); + await screen.findByRole("region", { name: "Timeline lane checkout" }); + await screen.findByRole("region", { name: "Logs at break", exact: true }); + const query = new URLSearchParams(window.location.search); + expect(query.get("center")).toBe(center); + expect(query.get("occurrence_id")).toBe("pinned-old"); + expect(api.kubernetesIssues).toHaveBeenCalledWith(expect.objectContaining({ context: "persisted", center, occurrence_id: "pinned-old" }), expect.anything()); + expect(screen.getByRole("slider", { name: "Log window start" })).toBeTruthy(); + await settle(); + expect(errors.mock.calls.filter(args => args.some(value => String(value).includes("Cannot update a component")))).toEqual([]); + } finally { + logs.mockRestore(); + errors.mockRestore(); + } + }); + + it("defers cache snapshot notifications and retires queued callbacks on unsubscribe", async () => { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + const batches = vi.spyOn(notifyManager, "batchCalls"); + const reads = vi.spyOn(client, "getQueryData"); + const { unmount } = renderPage(client); + await screen.findByRole("region", { name: "Cluster health" }); + await settle(); + vi.useFakeTimers(); + try { + const snapshotReads = () => reads.mock.calls.filter(([key]) => key.includes("kubernetes-changes") && key.includes("overview")).length; + const before = snapshotReads(); + // Even unrelated cache events reach this subscriber; never notify + // React synchronously while another component constructs its query. + client.setQueryData(["unrelated-notification"], 1); + expect(snapshotReads()).toBe(before); + expect(batches).toHaveBeenCalled(); + await act(async () => { await vi.advanceTimersByTimeAsync(1); }); + expect(snapshotReads()).toBeGreaterThan(before); + const queued = batches.mock.calls.at(-1)![0]; + client.setQueryData(["unrelated-notification"], 2); + unmount(); + reads.mockClear(); + // Exercise both an already queued notification and a stale retained + // callback: unsubscription must make the callback itself inert. + queued(); + await act(async () => { await vi.advanceTimersByTimeAsync(1); }); + expect(snapshotReads()).toBe(0); + } finally { + unmount(); + reads.mockRestore(); + batches.mockRestore(); + vi.useRealTimers(); + } + }); + + it("keeps mixed workload actions and the header in the same scrollbar width context", async () => { + const kinds = ["Deployment", "CronJob", "Pod", "ConfigMap"]; + vi.mocked(api.kubernetesWorkloads).mockResolvedValue({ items: kinds.map(kind => ({ + resource_id: kind === "ConfigMap" ? "core~v1~configmaps" : kind === "Pod" ? "core~v1~pods" : kind === "CronJob" ? "batch~v1~cronjobs" : "apps~v1~deployments", + kind, namespace: "shop", name: `${kind}-${"long-name-".repeat(20)}`, + })), counts: { Deployment: 1, CronJob: 1, Pod: 1, ConfigMap: 1 }, truncated: false }); + renderWorkloads(); + const inventory = await screen.findByRole("region", { name: "Workloads" }); + const rows = await within(inventory).findAllByRole("listitem"); + expect(rows).toHaveLength(4); + const header = inventory.querySelector(".kubernetes-workload-head")!; + expect(header.parentElement).toBe(rows[0].parentElement); + expect(header.classList.contains("kubernetes-workload-shell")).toBe(true); + for (const row of rows) { + expect(row.classList.contains("kubernetes-workload-shell")).toBe(true); + expect(row.children).toHaveLength(2); + expect(row.children[0].tagName).toBe("BUTTON"); + expect(row.children[0].querySelector("button")).toBeNull(); + expect(row.children[1].className).toBe(rows[0].children[1].className); + } + expect(within(rows[3]).queryByRole("button", { name: /Logs/ })).toBeNull(); + expect(within(rows[1]).queryByRole("button", { name: /Logs/ })).toBeNull(); + for (const row of [rows[0], rows[2]]) expect(within(row).getByRole("button", { name: /Logs/ }).parentElement).toBe(row.children[1]); + }); + + it("does not manufacture numeric kind badges without a scoped publication or borrow rows for a new filter", async () => { + let finish!: (value: Awaited>) => void; + vi.mocked(api.kubernetesWorkloads).mockReturnValueOnce(new Promise(resolve => { finish = resolve; })); + renderWorkloads(); + const inventory = await screen.findByRole("region", { name: "Workloads" }); + expect(within(inventory).getByText("Loading inventory")).toBeTruthy(); + const tabs = within(inventory).getByRole("tablist", { name: "Workload kind" }); + expect(tabs.querySelectorAll(".tabular-nums")).toHaveLength(0); + await act(async () => { finish({ items: [{ resource_id: "apps~v1~deployments", kind: "Deployment", namespace: "shop", name: "committed-row" }], counts: { Deployment: 1 }, truncated: false }); }); + await within(inventory).findByText("committed-row"); + vi.mocked(api.kubernetesWorkloads).mockReturnValueOnce(new Promise(() => {})); + fireEvent.click(within(tabs).getByRole("tab", { name: "CronJob" })); + expect(within(inventory).queryByText("committed-row")).toBeNull(); + expect(tabs.querySelectorAll(".tabular-nums")).toHaveLength(0); + }); + it("does not present denied workload counts as zero or an aggregate total", async () => { + vi.mocked(api.kubernetesWorkloads).mockResolvedValue({ items: [{ resource_id: "apps~v1~deployments", kind: "Deployment", namespace: "shop", name: "authorized-deployment" }], + counts: { Deployment: 1, Pod: 0 }, truncated: false, partial_failures: [{ resource_id: "core~v1~pods", class: "forbidden" }] }); + renderWorkloads(); + await screen.findByText("authorized-deployment"); + const inventory = screen.getByRole("region", { name: "Workloads" }); + const tabs = within(inventory).getByRole("tablist", { name: "Workload kind" }); + expect(within(tabs).getByRole("tab", { name: "Pod" }).querySelector(".tabular-nums")).toBeNull(); + expect(within(tabs).getByRole("tab", { name: "All" }).querySelector(".tabular-nums")).toBeNull(); + expect(within(tabs).getByRole("tab", { name: "Deployment" }).textContent).toContain("1"); + expect(within(inventory).getByText("Count unavailable")).toBeTruthy(); + }); + it("shows committed first-time timeout rows with omitted counts unknown and observed zero known", async () => { + const observed = "2026-08-30T12:00:00Z"; + vi.mocked(api.kubernetesSyncHealth).mockResolvedValue({ state: "error", partial: true, age_s: 18, + first_ready_at: observed, total_kinds: 6, ready_kinds: 5, failing_kinds: 1, pending_kinds: 0, truncated: false, + kinds: { CronJob: { state: "error", records: 0, partial: true, error: "timeout" } }, unreadable: [] }); + vi.mocked(api.kubernetesWorkloads).mockResolvedValue({ items: [{ resource_id: "apps~v1~deployments", kind: "Deployment", namespace: "shop", name: "committed-timeout-row" }], + counts: { Deployment: 1, StatefulSet: 0, DaemonSet: 0, Job: 0, Pod: 0 }, truncated: false, + sync: { state: "error", partial: true, age_s: 18, pending: null, pending_kinds: 0 }, + partial_failures: [{ resource_id: "batch~v1~cronjobs", class: "timeout" }] }); + renderWorkloads(); + await screen.findByText("committed-timeout-row"); + const inventory = screen.getByRole("region", { name: "Workloads" }); + const tabs = within(inventory).getByRole("tablist", { name: "Workload kind" }); + for (const kind of ["CronJob", "All"]) expect(within(tabs).getByRole("tab", { name: kind }).querySelector(".tabular-nums")).toBeNull(); + expect(within(tabs).getByRole("tab", { name: "Deployment" }).querySelector(".tabular-nums")?.textContent).toBe("1"); + expect(within(tabs).getByRole("tab", { name: "Job" }).querySelector(".tabular-nums")?.textContent).toBe("0"); + expect(within(inventory).getByText("Count unavailable")).toBeTruthy(); + expect(within(inventory).queryByText(/^\d+ total$/)).toBeNull(); + }); + it("recovers a typed expired inventory cursor explicitly without showing first-page rows under the later page", async () => { + let replacement = false; + vi.mocked(api.kubernetesWorkloads).mockImplementation(async options => { + if (options?.cursor) throw new ApiError(400, "Invalid query"); + return { items: [{ resource_id: "apps~v1~deployments", kind: "Deployment", namespace: "shop", name: replacement ? "replacement-page-one" : "old-page-one" }], + counts: { Deployment: 21 }, next: "expired-cursor", truncated: true }; + }); + renderWorkloads(); + await screen.findByText("old-page-one"); + const inventory = screen.getByRole("region", { name: "Workloads" }); + fireEvent.click(within(inventory).getByRole("button", { name: "Next page" })); + const restart = await within(inventory).findByRole("button", { name: "Restart inventory paging" }); + expect(within(inventory).queryByText("old-page-one")).toBeNull(); + expect(within(inventory).queryByText("No workloads in this scope.")).toBeNull(); + expect(within(inventory).queryByText("0 total")).toBeNull(); + replacement = true; + fireEvent.click(restart); + await within(inventory).findByText("replacement-page-one"); + expect(within(inventory).getByRole("button", { name: "Previous page" }).hasAttribute("disabled")).toBe(true); + expect(vi.mocked(api.kubernetesWorkloads).mock.calls.at(-1)?.[0].cursor).toBeUndefined(); + }); it("follows the coverage tip router link into the Agent tab without reloading", async () => { window.history.replaceState(null, "", "/agent/kubernetes?view=timeline&root_uid=root&namespace=shop"); const retained: KubernetesIssue = { root: { kind: "Deployment", name: "checkout", namespace: "shop", uid: "root" }, rule: "pod.crash_loop", severity: "critical", count: 1, broken: true, broke_at: "2026-10-17T12:00:00Z" }; - vi.mocked(api.kubernetesIssues).mockImplementation(async options => ({ items: options?.context === "persisted" ? [] : [retained], totals: {}, truncated: false, sync: { state: "ready", age_s: 0, partial: false }, ...(options?.context === "persisted" ? { root_context: { root: retained.root, occurrence: retained, retained: true as const } } : {}) })); + vi.mocked(api.kubernetesIssues).mockImplementation(async options => ({ items: options?.context === "persisted" ? [] : [retained], totals: {}, truncated: false, sync: { state: "ready", age_s: 0, partial: false }, ...(options?.context === "persisted" ? { selected_occurrence: retained, selection_status: "resolved" as const, root_context: { root: retained.root, occurrence: retained, retained: true as const } } : {}) })); vi.spyOn(api, "kubernetesBreakLogs").mockResolvedValue({ window: { from: "2026-10-17T11:55:00Z", to: "2026-10-17T12:02:00Z" }, pods: [], lines: [], markers: [], counts: { all: 0, errors: 0, warnings: 0 }, rate: [], patterns: [], sources: [], coverage: { state: "none", tip: true, reason: "pod_deleted", gaps: [] } }); const fields = { namespace: "namespace", pod: "pod", container: "container" }; vi.spyOn(api, "kubernetesAgentSettings").mockResolvedValue({ learning: { logs: false, metrics: false, traces: false, scope: { include: [], exclude: [] } }, break_logs: { linked_source: "", fields: { loki: fields, elasticsearch: fields, cloudwatchlogs: fields, graylog: fields, splunk: fields, signoz: fields } }, can_write: false, linked_source_missing: false, sources: [], detected: { workloads: 1, namespaces: 1, metrics_server: false, traces: null } }); @@ -619,7 +791,7 @@ describe("KubernetesPage", () => { expect(api.kubernetesGraph).not.toHaveBeenCalled(); expect(api.kubernetesNamespaces).not.toHaveBeenCalled(); expect(api.kubernetesWorkloads).toHaveBeenCalledOnce(); - expect(api.kubernetesWorkloads).toHaveBeenCalledWith(expect.objectContaining({ limit: 20 })); + expect(api.kubernetesWorkloads).toHaveBeenCalledWith(expect.objectContaining({ limit: 20 }), expect.any(AbortSignal)); }); it("keeps workloads full width and the sampled timestamp naturally left aligned", async () => { @@ -649,11 +821,12 @@ describe("KubernetesPage", () => { renderWorkloads(client); const inventory = await screen.findByRole("region", { name: "Workloads" }); await waitFor(() => expect(client.isFetching()).toBe(0)); - expect(within(inventory).getByText("137 total")).toBeTruthy(); + expect(within(inventory).getByText("Count unavailable")).toBeTruthy(); + expect(within(inventory).getByRole("tab", { name: "Deployment" }).textContent).toContain("137"); vi.mocked(api.kubernetesWorkloads).mockRejectedValueOnce(new ApiError(status, "Access denied")); await act(async () => { await client.invalidateQueries({ queryKey: ["kubernetes", "_", "kubernetes-workloads"] }); }); expect(await within(inventory).findByText("Inventory unavailable")).toBeTruthy(); - expect(within(inventory).queryByText("137 total")).toBeNull(); + expect(within(inventory).queryByText("Count unavailable")).toBeNull(); expect(within(inventory).queryByText("137", { exact: true })).toBeNull(); expect(within(inventory).queryByText("private-api")).toBeNull(); expect(within(inventory).queryByText(/Some evidence could not be collected/)).toBeNull(); @@ -1338,7 +1511,7 @@ describe("KubernetesPage", () => { renderWorkloads(); const inventory = await screen.findByRole("region", { name: "Workloads" }); expect(await within(inventory).findAllByRole("button", { name: /Select Deployment/ })).toHaveLength(20); - expect(api.kubernetesWorkloads).toHaveBeenCalledWith(expect.objectContaining({ limit: 20 })); + expect(api.kubernetesWorkloads).toHaveBeenCalledWith(expect.objectContaining({ limit: 20 }), expect.any(AbortSignal)); }); it("renders health, partial visibility, and cluster-scoped nodes", async () => { @@ -1377,11 +1550,11 @@ describe("KubernetesPage", () => { fireEvent.change(screen.getByLabelText("Resource name"), { target: { value: "nightly" } }); expect(await screen.findByText("nightly-cleanup")).toBeTruthy(); expect(screen.queryByText("checkout-api")).toBeNull(); - await waitFor(() => expect(api.kubernetesWorkloads).toHaveBeenCalledWith(expect.objectContaining({ q: "nightly", limit: 20 }))); + await waitFor(() => expect(api.kubernetesWorkloads).toHaveBeenCalledWith(expect.objectContaining({ q: "nightly", limit: 20 }), expect.any(AbortSignal))); fireEvent.focus(screen.getByLabelText("Workload namespace")); fireEvent.click(await screen.findByRole("option", { name: "payments" })); expect(await screen.findByText("No matching workloads.")).toBeTruthy(); - expect(api.kubernetesWorkloads).toHaveBeenLastCalledWith(expect.objectContaining({ namespace: "payments", q: "nightly", limit: 20 })); + expect(api.kubernetesWorkloads).toHaveBeenLastCalledWith(expect.objectContaining({ namespace: "payments", q: "nightly", limit: 20 }), expect.any(AbortSignal)); expect(api.kubernetesSearch).not.toHaveBeenCalled(); }); @@ -1487,8 +1660,8 @@ describe("KubernetesPage", () => { // One page has nowhere to go, so the pager is gone rather than disabled. expect(within(workloads).queryByRole("button", { name: "Next page" })).toBeNull(); expect(within(workloads).queryByRole("navigation", { name: "Page navigation" })).toBeNull(); - expect(api.kubernetesWorkloads).toHaveBeenCalledWith(expect.objectContaining({ cursor: "workload-next", limit: 20 })); - expect(api.kubernetesWorkloads).toHaveBeenLastCalledWith(expect.objectContaining({ kind: "Job", cursor: undefined, limit: 20 })); + expect(api.kubernetesWorkloads).toHaveBeenCalledWith(expect.objectContaining({ cursor: "workload-next", limit: 20 }), expect.any(AbortSignal)); + expect(api.kubernetesWorkloads).toHaveBeenLastCalledWith(expect.objectContaining({ kind: "Job", cursor: undefined, limit: 20 }), expect.any(AbortSignal)); }); it("fetches only the current server page for workloads and nodes without querying warning events", async () => { @@ -2246,6 +2419,69 @@ describe("KubernetesPage sync gate", () => { } }); + it.each([ + ["typed not-ready", "ready"], ["malformed success", "ready"], + ["typed not-ready", "degraded"], ["typed not-ready", "failed"], + ])("bounds late %s recovery under %s health to five scoped retries, then offers an explicit retry instead of polling forever", async (response, state) => { + window.history.replaceState(null, "", "/agent/kubernetes?view=resources"); + vi.mocked(api.kubernetesOverview).mockResolvedValue(healthyOverview); + const recoveryReport = { ...report(state === "failed" ? "error" : "ready", { + Pod: state === "ready" ? readyKind : { ...readyKind, state: state === "failed" ? "error" : "ready", partial: true, error: state === "failed" ? "unavailable" : "timeout" }, + }, { failing: state === "ready" ? 0 : 1 }), first_ready_at: published }; + vi.mocked(api.kubernetesSyncHealth).mockResolvedValue(recoveryReport); + let fail!: (error: ApiError) => void; + let finish!: (value: Awaited>) => void; + vi.mocked(api.kubernetesWorkloads).mockReturnValueOnce(new Promise((resolve, reject) => { finish = resolve; fail = reject; })); + const { container } = renderPage(); + await waitFor(() => expect(api.kubernetesWorkloads).toHaveBeenCalledTimes(1)); + if (state !== "ready") expect(container.querySelector(`[data-sync-strip="${state}"]`)).toBeTruthy(); + vi.useFakeTimers(); + try { + const error = new ApiError(503, "warming", { code: "inventory_not_ready", retryable: true }); + const malformed = { items: [], counts: {}, truncated: false, sync: { state: "", age_s: 0, partial: false } }; + if (response === "typed not-ready") vi.mocked(api.kubernetesWorkloads).mockRejectedValue(error); + else vi.mocked(api.kubernetesWorkloads).mockResolvedValue(malformed); + await act(async () => { + if (response === "typed not-ready") fail(error); + else finish(malformed); + await vi.advanceTimersByTimeAsync(20); + }); + for (let retry = 0; retry < 6; retry++) await act(async () => { await vi.advanceTimersByTimeAsync(1_050); }); + expect(api.kubernetesWorkloads).toHaveBeenCalledTimes(6); + expect(screen.getByRole("button", { name: "Retry inventory" })).toBeTruthy(); + expect(screen.getByText("Inventory unavailable")).toBeTruthy(); + expect(screen.queryByText("Loading inventory")).toBeNull(); + expect(container.querySelector("[data-preparing-panel]")).toBeNull(); + expect(screen.queryByText("0 total")).toBeNull(); + await act(async () => { await vi.advanceTimersByTimeAsync(30_000); }); + expect(api.kubernetesWorkloads).toHaveBeenCalledTimes(6); + vi.mocked(api.kubernetesWorkloads).mockResolvedValue({ items: [], counts: { Deployment: 0, StatefulSet: 0, DaemonSet: 0, Job: 0, CronJob: 0, Pod: 0 }, truncated: false, sync: { state: "ready", age_s: 0, partial: false } }); + fireEvent.click(screen.getByRole("button", { name: "Retry inventory" })); + await act(async () => { await vi.advanceTimersByTimeAsync(20); }); + expect(screen.getByText("No workloads in this scope.")).toBeTruthy(); + expect(screen.getByText("0 total")).toBeTruthy(); + } finally { vi.useRealTimers(); } + }); + + it.each(["cronjobs", "batch~v1~cronjobs"])("keeps pending %s counts unknown until the scope is actually listed", async pending => { + vi.mocked(api.kubernetesOverview).mockResolvedValue(healthyOverview); + vi.mocked(api.kubernetesSyncHealth).mockResolvedValue({ ...report("ready", { Pod: readyKind }), first_ready_at: published }); + const listing = { items: [], counts: { Deployment: 2, CronJob: 0 }, truncated: false, sync: { state: "ready", age_s: 0, partial: false, pending: [pending] } }; + vi.mocked(api.kubernetesWorkloads).mockResolvedValue(listing); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + renderWorkloads(client); + await screen.findByText("Count unavailable"); + const kinds = screen.getByRole("tablist", { name: "Workload kind" }); + expect(within(kinds).getByRole("tab", { name: "CronJob" }).textContent).toBe("CronJob"); + expect(within(kinds).getByRole("tab", { name: /Deployment/ }).textContent).toBe("Deployment2"); + expect(within(kinds).getByRole("tab", { name: "All" }).textContent).toBe("All"); + expect(screen.queryByText("2 total")).toBeNull(); + vi.mocked(api.kubernetesWorkloads).mockResolvedValue({ ...listing, counts: { Deployment: 2, StatefulSet: 0, DaemonSet: 0, Job: 0, CronJob: 0, Pod: 0 }, sync: { ...listing.sync, pending: [] } }); + await act(async () => { await client.invalidateQueries({ queryKey: ["kubernetes", "_", "kubernetes-workloads"] }); }); + await screen.findByText("2 total"); + expect(within(kinds).getByRole("tab", { name: /CronJob/ }).textContent).toBe("CronJob0"); + }); + it.each([ ["malformed success", false], ["malformed success", true], ["typed not-ready", false], ["typed not-ready", true], @@ -2283,14 +2519,14 @@ describe("KubernetesPage sync gate", () => { expect(screen.queryByText("No workloads in this scope.")).toBeNull(); expect(screen.queryByText("0 total")).toBeNull(); if (cached) expect(screen.getByText("committed-api")).toBeTruthy(); - else expect(screen.getByText("Inventory unavailable")).toBeTruthy(); + else expect(screen.getByText(response === "typed not-ready" ? "Loading inventory" : "Inventory unavailable")).toBeTruthy(); await act(async () => { await vi.advanceTimersByTimeAsync(1_000); }); expect(screen.getByText("recovered-api")).toBeTruthy(); expect(api.kubernetesWorkloads).toHaveBeenCalledTimes(2); await act(async () => { await vi.advanceTimersByTimeAsync(3_000); }); expect(api.kubernetesWorkloads).toHaveBeenCalledTimes(2); // A real complete-empty publication from the existing stream replaces rows. - vi.mocked(api.kubernetesWorkloads).mockResolvedValue({ ...malformed, sync: committed.sync }); + vi.mocked(api.kubernetesWorkloads).mockResolvedValue({ ...malformed, counts: { Deployment: 0, StatefulSet: 0, DaemonSet: 0, Job: 0, CronJob: 0, Pod: 0 }, sync: committed.sync }); await act(async () => { emit({ event: "records", data: "{}" }); await vi.advanceTimersByTimeAsync(800); }); expect(screen.queryByText("recovered-api")).toBeNull(); expect(screen.getByText("No workloads in this scope.")).toBeTruthy(); @@ -2369,6 +2605,22 @@ describe("KubernetesPage sync gate", () => { expect(client.getQueryData(lateKey)).toEqual(currentGraph); expect(screen.queryByText("protected-api")).toBeNull(); expect(screen.queryByRole("dialog", { name: "Details panel" })).toBeNull(); + // An old callback can outlive both the denial and the successful refresh. + // It must not invalidate or overwrite the new authority's publication. + const reauthorizedReads = vi.mocked(api.kubernetesWorkloads).mock.calls.length; + vi.useFakeTimers(); + try { + await act(async () => { + emitLog({ event: "line", container: "app", text: "retired after refresh" }); + oldStream[0]({ event: "sync", data: JSON.stringify({ state: "error", partial: true }) }); + oldStream[0]({ event: "records", data: "{}" }); + await vi.advanceTimersByTimeAsync(800); + }); + expect(screen.queryByText("retired after refresh")).toBeNull(); + expect(screen.getByText("reauthorized-api")).toBeTruthy(); + expect(client.getQueryData(lateKey)).toEqual(currentGraph); + expect(api.kubernetesWorkloads).toHaveBeenCalledTimes(reauthorizedReads); + } finally { vi.useRealTimers(); } }); it("recovers through report polling after correlated overview denial and a held cycle-authority failure", async () => { diff --git a/ui/src/pages/KubernetesPage.tsx b/ui/src/pages/KubernetesPage.tsx index e5d21c26..a79bdef5 100644 --- a/ui/src/pages/KubernetesPage.tsx +++ b/ui/src/pages/KubernetesPage.tsx @@ -9,7 +9,7 @@ import { type Ref, type ReactNode, } from "react"; -import { useQueries, useQuery, useQueryClient, type Query } from "@tanstack/react-query"; +import { notifyManager, useQueries, useQuery, useQueryClient, type Query } from "@tanstack/react-query"; import { Link, useLocation, useNavigate, useSearchParams } from "react-router-dom"; import { KubernetesFirstReadyContext, useBrokenIssues, useKubernetesCluster, useKubernetesQuery } from "@/lib/useKubernetesCluster"; import { KubernetesClusterProvider } from "@/components/KubernetesClusterProvider"; @@ -38,6 +38,7 @@ import { } from "lucide-react"; import { api, ApiError, type KubernetesChangesPage, type KubernetesCrashEvent, type KubernetesIndexStatus, type KubernetesOverview, type KubernetesSyncHealthReport, type KubernetesResource, type KubernetesUsage } from "@/lib/api"; import { kubernetesCatalogRefreshInterval } from "@/lib/kubernetesFirstSync"; +import { pendingKindNames } from "@/lib/kubernetesPending"; import { PodLogViewer } from "@/components/PodLogViewer"; import { ResourceLogsButton } from "@/components/ResourceLogsButton"; import { canStreamLogs } from "@/lib/kubernetesLogTargets"; @@ -521,41 +522,78 @@ function KubernetesDashboard() { * degraded data need a strip; routine syncing uses the existing top bar. */ const clusterQueriesEnabled = !authorityLost && clusterReachable && (health.state === "ready" || health.firstReadyAt !== undefined); + const [exhaustedReads, setExhaustedReads] = useState>(new Set()); useEffect(() => { - if (!clusterQueriesEnabled || health.state !== "ready" || !health.firstReadyAt) return; + setExhaustedReads(new Set()); + if (!clusterQueriesEnabled || !health.firstReadyAt) return; // A pre-publication response may finish after readiness, even as a malformed // empty success or typed not-ready error. Recheck only those scoped reads at // most once a second; settled data, faults and denials do not keep polling. + const attempts = new WeakMap(); const filters = { queryKey: key(), predicate: (query: Query) => query.isActive() && query.state.fetchStatus === "idle" && !["kubernetes-overview", "kubernetes-sync-health"].includes(String(query.queryKey[2])) - && kubernetesReadNeedsRecovery(query.state.data, query.state.error), + && kubernetesReadNeedsRecovery(query.state.data, query.state.error) + && (attempts.get(query) ?? 0) < 5, + }; + const retryEligible = () => { + const admitted = new Set(queryClient.getQueryCache().findAll(filters)); + for (const query of admitted) attempts.set(query, (attempts.get(query) ?? 0) + 1); + void queryClient.invalidateQueries({ ...filters, predicate: query => admitted.has(query) }, { cancelRefetch: false }); }; let timer: ReturnType | undefined; const schedule = () => { + const exhausted = new Set(queryClient.getQueryCache().findAll({ queryKey: key(), + predicate: query => query.isActive() && query.state.fetchStatus === "idle" + && (attempts.get(query) ?? 0) >= 5 && kubernetesReadNeedsRecovery(query.state.data, query.state.error), + }).map(query => JSON.stringify(query.queryKey))); + setExhaustedReads(previous => previous.size === exhausted.size && [...previous].every(value => exhausted.has(value)) ? previous : exhausted); if (!queryClient.getQueryCache().findAll(filters).length) { clearTimeout(timer); timer = undefined; } else if (timer === undefined) { timer = setTimeout(() => { timer = undefined; - void queryClient.invalidateQueries(filters, { cancelRefetch: false }); + retryEligible(); schedule(); }, 1_000); } }; - void queryClient.invalidateQueries(filters, { cancelRefetch: false }); - const unsubscribe = queryClient.getQueryCache().subscribe(schedule); + retryEligible(); + // Query observers can update the cache during a child render. Defer the + // exhaustion state update until that render finishes, and fence unmounts. + let disposed = false; + let scheduled = false; + const unsubscribe = queryClient.getQueryCache().subscribe(() => { + if (scheduled) return; + scheduled = true; + queueMicrotask(() => { + scheduled = false; + if (!disposed) schedule(); + }); + }); schedule(); return () => { + disposed = true; unsubscribe(); clearTimeout(timer); }; - }, [clusterQueriesEnabled, health.state, health.firstReadyAt, key, queryClient]); + }, [clusterQueriesEnabled, health.firstReadyAt, key, queryClient]); const brokenIssues = useBrokenIssues(view === "overview" && clusterQueriesEnabled && !overviewAccessDenied); const overviewChangesKey = useMemo(() => key("kubernetes-changes", "overview", overview.data?.cluster_id), [key, overview.data?.cluster_id]); - const subscribeOverviewChanges = useCallback((notify: () => void) => queryClient.getQueryCache().subscribe(notify), [queryClient]); + const subscribeOverviewChanges = useCallback((notify: () => void) => { + let active = true; + // Query construction in a child render also emits cache events. Match + // query observers' deferred notifications instead of updating the parent + // synchronously, and retire queued notifications when unsubscribing. + const notifyAfterRender = notifyManager.batchCalls(() => { if (active) notify(); }); + const unsubscribe = queryClient.getQueryCache().subscribe(notifyAfterRender); + return () => { + active = false; + unsubscribe(); + }; + }, [queryClient]); const getOverviewChanges = useCallback(() => queryClient.getQueryData(overviewChangesKey), [queryClient, overviewChangesKey]); const overviewChanges = useSyncExternalStore(subscribeOverviewChanges, getOverviewChanges, getOverviewChanges); @@ -578,16 +616,18 @@ function KubernetesDashboard() { const workloadPaging = useCursorPagination(`workloads:${workloadNamespace}:${name.trim()}:${workloadKind}`); const workloads = useQuery({ queryKey: ["kubernetes-workloads", workloadNamespace, name.trim(), workloadKind, workloadPaging.cursor], - queryFn: () => api.kubernetesWorkloads({ + queryFn: ({ signal }) => api.kubernetesWorkloads({ namespace: workloadNamespace === "All" ? undefined : workloadNamespace, kind: workloadKind === "All" ? undefined : workloadKind, q: name.trim() || undefined, limit: 20, cursor: workloadPaging.cursor, - }), + }, signal), enabled: clusterQueriesEnabled, retry: false, - }); + }, syncReport.isSuccess && !authorityLost && !scopeChanged + && !syncReport.data?.unreadable?.some(row => workloadKindOptions.includes(row.kind)) + ? firstReadyAt(syncReport.data) : undefined); const nodePaging = useCursorPagination("nodes"); const nodes = useQuery({ queryKey: ["kubernetes-nodes", nodePaging.cursor], @@ -813,7 +853,19 @@ function KubernetesDashboard() { const visibleResources = asArray(workloads.data?.items); const nodeItems = asArray(nodes.data?.items); const selectedNodePods = asArray(nodePods.data?.items); - const workloadCounts = workloads.data?.counts ?? {}; + const pendingWorkloadKinds = pendingKindNames(workloads.data?.sync?.pending); + const unknownWorkloadCounts = workloadKindOptions.filter(kind => + typeof workloads.data?.counts?.[kind] !== "number" + || pendingWorkloadKinds.has(kind) + || + syncReport.data?.unreadable?.some(row => row.kind === kind) + || workloads.data?.partial_failures?.some(failure => + ["forbidden", "unauthorized", "namespace_scope", "credential_unavailable"].includes(failure.class) + && (!failure.resource_id || kubernetesQueryUsesKinds([failure.resource_id], undefined, new Set([kind]))))); + const workloadCounts = Object.fromEntries(Object.entries(workloads.data?.counts ?? {}).filter(([kind]) => !unknownWorkloadCounts.includes(kind))); + const workloadRecoveryExhausted = exhaustedReads.has(JSON.stringify(key("kubernetes-workloads", workloadNamespace, name.trim(), workloadKind, workloadPaging.cursor))); + const workloadWarming = !workloadRecoveryExhausted && (workloads.unpublished || kubernetesReadNeedsRecovery(workloads.data, workloads.error)); + const workloadCursorExpired = Boolean(workloadPaging.cursor && workloads.error instanceof ApiError && workloads.error.status === 400); const workloadTotal = workloadKindOptions.reduce((sum, kind) => sum + (workloadCounts[kind] ?? 0), 0); useEffect(() => { @@ -945,7 +997,7 @@ function KubernetesDashboard() { {multiple && <>Clusters} } trailing={ - {workloadsAccessDenied || !workloads.data ? "Inventory unavailable" : `${workloadTotal} total`} + {workloadsAccessDenied ? "Inventory unavailable" : !workloads.data ? workloads.isPending || workloadWarming ? "Loading inventory" : "Inventory unavailable" : workloads.data.counts && !unknownWorkloadCounts.length ? `${workloadTotal} total` : "Count unavailable"} } > @@ -1164,8 +1216,10 @@ function KubernetesDashboard() { setWorkloadNamespace(namespace || "All")} /> - - {workloads.isPending && ( + + {workloadCursorExpired &&

This inventory page expired or its filters changed. Restart paging.

} + {workloads.data && workloads.isError && !workloadsAccessDenied && !workloadCursorExpired &&

Resource inventory refresh is unavailable. Showing the last authorized observation.

} + {(workloads.isPending || workloadWarming && !workloads.data) && (
@@ -1175,9 +1229,10 @@ function KubernetesDashboard() { )} - {(workloadsAccessDenied || !workloads.data && !workloads.isPending) && ( + {(workloadsAccessDenied || !workloads.data && !workloads.isPending && !workloadWarming && !workloadCursorExpired) && (

Resource inventory is unavailable. + {workloadRecoveryExhausted && }

)} {/* The rows already in hand survive a background resync: once this @@ -1185,19 +1240,22 @@ function KubernetesDashboard() { show is replaced by the preparing surface. */} {workloads.data && (Boolean(health.firstReadyAt) || workloads.data.sync?.state !== "syncing") && !workloadsAccessDenied && (
+
{visibleResources.length > 0 && ( -
+
+
Name Namespace Kind Readiness / status Select
+
)} -
{visibleResources.map((item) => ( ; selected: string; onSelect: (kind: string) => void; + allKnown?: boolean; }) { const options = ["All", ...kinds]; return ( @@ -1500,7 +1560,7 @@ function KindTabs({ key={kind} > {kind} - + {(kind === "All" ? allKnown && Object.keys(counts).length > 0 : counts[kind] !== undefined) && } ); })} @@ -1769,7 +1829,7 @@ function ResourceRow({ const unhealthy = condition?.status === "False"; const status = workloadStatus(item); return ( -
+
- {onLogs &&
} +
{onLogs && }
); } diff --git a/ui/src/pages/KubernetesResourceDetail.css b/ui/src/pages/KubernetesResourceDetail.css index fcea62eb..2501a6f8 100644 --- a/ui/src/pages/KubernetesResourceDetail.css +++ b/ui/src/pages/KubernetesResourceDetail.css @@ -635,6 +635,11 @@ align-items: center; gap: 12px; } +.kubernetes-workload-shell { + display: grid; + grid-template-columns: minmax(0, 1fr) 3.5rem; +} +.kubernetes-workload-head { display: none; } .kubernetes-list-head { display: none; grid-template-columns: minmax(0, 1.4fr) minmax(0, 1fr) 7rem minmax(7rem, 1fr) 1.5rem; @@ -655,6 +660,7 @@ grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); } @container (min-width: 34rem) { + .kubernetes-workload-head { display: grid; } .kubernetes-list-head { display: grid; } .kubernetes-list-toolbar { grid-template-columns: minmax(0, 1fr) 14rem; } .kubernetes-list-row--workload { grid-template-columns: minmax(0, 1.4fr) minmax(0, 1fr) 7rem minmax(7rem, 1fr) 1.5rem; } diff --git a/ui/tests/e2e/kubernetes-overview.spec.ts b/ui/tests/e2e/kubernetes-overview.spec.ts index 3f8760e2..95da90c2 100644 --- a/ui/tests/e2e/kubernetes-overview.spec.ts +++ b/ui/tests/e2e/kubernetes-overview.spec.ts @@ -469,6 +469,11 @@ async function expectNoTopologyPagination(panel: Locator) { } const screenshotDir = process.env.E2E_KUBERNETES_CAPTURE_DIR || captureDir(); +// Assertion-only reruns retain inspected captures for unaffected visual inputs. +const captureUnchanged = process.env.E2E_CAPTURE_UNCHANGED !== "0"; +if (process.env.E2E_NATIVE_SCROLLBARS === "1") { + test.use({ launchOptions: { args: ["--disable-features=OverlayScrollbar,FluentOverlayScrollbar"] } }); +} async function fitOverviewCapture(page: Page) { const measurements = []; @@ -2429,6 +2434,7 @@ test.describe("Pod log streaming fixtures", () => { test.beforeEach(async ({ page }, testInfo) => { const overviewRefresh = testInfo.title.startsWith("Overview refresh"); const breakTimeline = testInfo.title.startsWith("Break timeline"); + const focusRegression = testInfo.title.includes("selected marker removal") || testInfo.title.includes("external controls retain focus"); const lightTheme = testInfo.title.includes("light theme"); const fullOverview = testInfo.title === "Break timeline full populated dark Overview documentation"; const guideAlerts = { enabled: false, level: "critical", enabled_at: "", can_write: true, available: true, reason: "", links_ready: true, channels_enabled: true }; @@ -2586,6 +2592,8 @@ test.describe("Pod log streaming fixtures", () => { } if (pathname === "/api/admin/config/agent") return route.fulfill({ json: { enable: breakTimeline, mode: "shadow", sources: [], ai: { enable: breakTimeline }, catalog: {}, miner: {}, regex: { rules: [] }, redaction: {}, service_patterns: [] } }); if (multiClusterBreak && pathname === "/api/admin/kubernetes/clusters") return route.fulfill({ json: { multiple: true, clusters: ["east", "west"].map((id) => ({ id, provider: "generic", health: "attention", nodes: 12, ready_nodes: 12, pods: 28, running_pods: 19, warnings: 26, broken: 3, ...(testInfo.title.includes("fleet documentation") ? { last_change_at: id === "east" ? at(9) : "invalid-timestamp" } : {}), sync: { state: "ready" }, observed_at: new Date().toISOString() })) } }); + if (focusRegression && pathname === "/api/admin/kubernetes/clusters") return route.fulfill({ json: { multiple: false, clusters: [] } }); + if (focusRegression && pathname === "/api/admin/kubernetes/namespaces") return route.fulfill({ json: { items: ["shop", "payments"], sync: { state: "ready", partial: false, age_s: 0 } } }); if (breakTimeline && pathname === "/api/admin/kubernetes/overview") return route.fulfill({ json: { connector: "kubernetes", cluster_id: "fixture", observed_at: new Date().toISOString(), nodes: 12, ready_nodes: 12, pods: 28, running_pods: 19, namespaces: 3, active_namespaces: 3, workloads: 4, warnings: 26, truncated: diagnostics, omitted_categories: diagnostics ? ["ConfigMap"] : [], partial_failures: failures } }); if (breakTimeline && pathname === "/api/admin/kubernetes/crashes") { const cursor = url.searchParams.get("cursor"); @@ -2613,6 +2621,7 @@ test.describe("Pod log streaming fixtures", () => { return route.fulfill({ json: result }); } if (breakTimeline && pathname === "/api/admin/kubernetes/issues") { + if (focusRegression && url.searchParams.get("context") === "persisted") return route.fulfill({ json: { items: [], totals: {}, truncated: false, sync: { state: "ready", partial: false, age_s: 0 }, root_context: { root: roots[0].root, occurrence: roots[0], retained: true }, selected_occurrence: roots[0], selection_status: "resolved" } }); const rootLookup = url.searchParams.has("root_uid"); if (denialEdge && rootLookup && (++edgeRootReads > 1 || denialEdge[1] === "initial")) return route.fulfill({ status: Number(denialEdge[2]), json: { error: "Synthetic root lookup denied" } }); const cart: KubernetesIssue = { root: { kind: "Deployment", namespace: "shop", name: "cart", uid: "cart-uid" }, rule: "event.warning", severity: "warning", count: 1, first_seen: at(1), history: "gap" }; @@ -3802,61 +3811,97 @@ test.describe("Pod log streaming fixtures", () => { }); test(`Break timeline selected marker removal restores chart focus ${viewport.name} dark`, async ({ page }) => { + const warnings: string[] = []; + page.on("console", message => { if (message.type() === "error" && message.text().includes("Cannot update a component")) warnings.push(message.text()); }); await page.setViewportSize(viewport); - await openApp(page, "/agent/kubernetes?view=timeline"); + await openApp(page, "/agent/kubernetes?view=timeline&root_uid=checkout-uid&namespace=shop"); const chart = page.getByRole("region", { name: "Resource timeline chart", exact: true }); const marker = chart.locator('[data-marker="change"]').first(); await marker.focus(); await page.keyboard.press("Enter"); const details = page.getByRole("region", { name: "Timeline event details", exact: true }); await expect(details.getByRole("heading", { name: "Timeline event details", exact: true })).toBeFocused(); - for (const endpoint of ["changes", "issues"]) { - await page.route(`**/api/admin/kubernetes/${endpoint}*`, (route) => route.fulfill({ json: { items: [], totals: {}, sync: { state: "ready", partial: false, age_s: 0 } } })); - } + await page.route("**/api/admin/kubernetes/changes*", (route) => route.fulfill({ json: { items: [], sync: { state: "ready", partial: false, age_s: 0 } } })); await page.getByRole("button", { name: "Refresh Kubernetes data", exact: true }).click(); await expect(details).toHaveCount(0); await expect(chart.locator('[data-marker="change"]')).toHaveCount(0); await expect(chart).toBeFocused(); expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + expect(warnings).toEqual([]); }); test(`Break timeline external controls retain focus across filters and polling ${viewport.name} dark`, async ({ page }) => { + const warnings: string[] = []; + page.on("console", message => { if (message.type() === "error" && message.text().includes("Cannot update a component")) warnings.push(message.text()); }); await page.setViewportSize(viewport); await page.clock.install({ time: new Date() }); - await page.route("**/api/admin/kubernetes/issues*", (route) => route.fulfill({ json: { items: [], totals: {}, sync: { state: "ready", partial: false, age_s: 0 } } })); - await openApp(page, "/agent/kubernetes?view=timeline"); + await openApp(page, "/agent/kubernetes?view=timeline&root_uid=checkout-uid&namespace=shop"); const chart = page.getByRole("region", { name: "Resource timeline chart", exact: true }); const marker = chart.locator('[data-marker="change"]').first(); const details = page.getByRole("region", { name: "Timeline event details", exact: true }); - const namespace = page.getByRole("textbox", { name: "Timeline namespace", exact: true }); - const kind = page.getByRole("combobox", { name: "Timeline kind", exact: true }); + const range = page.getByRole("group", { name: "Timeline window", exact: true }).getByRole("button", { name: "15m", exact: true }); await marker.focus(); await page.keyboard.press("Enter"); await expect(details.getByRole("heading", { name: "Timeline event details", exact: true })).toBeFocused(); - await namespace.focus(); - await namespace.pressSequentially("missing"); - await expect(namespace).toHaveValue("missing"); - await expect(namespace).toBeFocused(); - await expect(chart.locator('[data-marker="change"]')).toHaveCount(0); - await expect(details).toHaveCount(0); - await namespace.fill(""); - await marker.focus(); + await range.focus(); await page.keyboard.press("Enter"); - await kind.focus(); - await kind.selectOption("Deployment"); + await expect(range).toHaveAttribute("aria-pressed", "true"); + await expect(range).toBeFocused(); await expect(details).toHaveCount(0); - await expect(kind).toBeFocused(); await marker.focus(); await page.keyboard.press("Enter"); - await namespace.focus(); + const refresh = page.getByRole("button", { name: "Refresh Kubernetes data", exact: true }); + await refresh.focus(); const previousUntil = await chart.getAttribute("data-until"); - for (const endpoint of ["changes", "issues"]) { - await page.route(`**/api/admin/kubernetes/${endpoint}*`, (route) => route.fulfill({ json: { items: [], totals: {}, sync: { state: "ready", partial: false, age_s: 0 } } })); - } + await page.route("**/api/admin/kubernetes/changes*", (route) => route.fulfill({ json: { items: [], sync: { state: "ready", partial: false, age_s: 0 } } })); + await page.keyboard.press("Enter"); + await expect(details).toHaveCount(0); + await expect(chart.locator('[data-marker="change"]')).toHaveCount(0); + await expect(refresh).toBeFocused(); await page.clock.fastForward(30_000); - await expect(chart).not.toHaveAttribute("data-until", previousUntil!); + // Root investigations pin their range; polling must not move that bracket. + await expect(chart).toHaveAttribute("data-until", previousUntil!); await expect(chart.locator('[data-marker="change"]')).toHaveCount(0); await expect(details).toHaveCount(0); + await expect(refresh).toBeFocused(); + + // Namespace/kind filters now belong to the unfocused paginated list. + let removed = false; + const reads: URL[] = []; + await page.route("**/api/admin/kubernetes/timeline*", route => { + const url = new URL(route.request().url()); + reads.push(url); + const entry = { type: "change", id: "focus-change", at: new Date().toISOString(), change: { id: "focus-change", at: new Date().toISOString(), cluster: "fixture", kind: "Deployment", namespace: "shop", name: "checkout", uid: "checkout-uid", type: "image_changed" } }; + const items = removed || url.searchParams.get("namespace") === "payments" ? [] : [entry]; + return route.fulfill({ json: { items, total: items.length, counts: { change: items.length, crash: 0, break: 0 }, crash_count: 0, since: url.searchParams.get("since"), until: url.searchParams.get("until"), gaps: [], sync: { state: "ready", partial: false, age_s: 0 } } }); + }); + await page.getByRole("link", { name: "Clear root filter", exact: true }).click(); + const list = page.getByRole("list", { name: "Breaks and changes", exact: true }); + await expect(list.getByRole("listitem")).toHaveCount(1); + const namespace = page.getByRole("combobox", { name: "Timeline namespace", exact: true }); + await namespace.focus(); + await namespace.fill("payments"); + await expect(page.getByRole("option", { name: "payments", exact: true })).toBeVisible(); + await page.keyboard.press("Enter"); + await expect(list.getByRole("listitem")).toHaveCount(0); + await expect(namespace).toBeFocused(); + await namespace.click(); + await namespace.fill("shop"); + await expect(page.getByRole("option", { name: "shop", exact: true })).toBeVisible(); + await page.keyboard.press("Enter"); + await expect(list.getByRole("listitem")).toHaveCount(1); + const kind = page.getByRole("combobox", { name: "Timeline kind", exact: true }); + await kind.focus(); + await kind.selectOption("Deployment"); + await expect(kind).toBeFocused(); + await expect.poll(() => reads.at(-1)?.searchParams.get("kind")).toBe("Deployment"); + await namespace.focus(); + removed = true; + const listUntil = reads.at(-1)?.searchParams.get("until"); + await page.clock.fastForward(30_000); + await expect.poll(() => reads.at(-1)?.searchParams.get("until")).not.toBe(listUntil); + await expect(list.getByRole("listitem")).toHaveCount(0); await expect(namespace).toBeFocused(); + expect(warnings).toEqual([]); }); test(`Break timeline label and stack geometry ${viewport.name} dark`, async ({ page }) => { await page.setViewportSize(viewport); @@ -4409,6 +4454,553 @@ test.describe("Pod log streaming fixtures", () => { } }); +test.describe("Workloads and focused investigation worst-case browser gate", () => { + test.skip(process.env.E2E_UI_FIXTURES !== "1", "deterministic intercepted UI evidence, not live collection"); + + for (const width of [1440, 390]) { + test(`published never-listed timeout, last-good count and successful empty zero ${width}`, async ({ page }, testInfo) => { + await page.setViewportSize({ width, height: 900 }); + await page.addInitScript(() => localStorage.setItem("versus.theme", "dark")); + const observed = new Date().toISOString(); + let phase: "never-listed" | "known" | "later-timeout" | "empty" = "never-listed"; + const timedOut = () => phase === "never-listed" || phase === "later-timeout"; + const sync = () => ({ state: timedOut() ? "error" : "ready", partial: timedOut(), age_s: 18, + // The published timeout trace has no pending CronJob. + pending: null, pending_kinds: 0 }); + const partials = () => [ + ...(timedOut() ? [{ resource_id: "batch~v1~cronjobs", class: "timeout" }] : []), + ]; + const resources = [ + { resource_id: "apps~v1~deployments", kind: "Deployment", namespace: "shop", name: "only-a", uid: "only-a" }, + { resource_id: "apps~v1~deployments", kind: "Deployment", namespace: "shop", name: "web", uid: "web-v1" }, + { resource_id: "core~v1~pods", kind: "Pod", namespace: "shop", name: "only-a-0", uid: "only-a-pod" }, + { resource_id: "core~v1~pods", kind: "Pod", namespace: "shop", name: "web-0", uid: "web-v1-pod" }, + { resource_id: "apps~v1~statefulsets", kind: "StatefulSet", namespace: "shop", name: "db", uid: "db-v1" }, + ]; + const payloads: unknown[] = []; + await page.route("**/api/**", async route => { + const pathname = new URL(route.request().url()).pathname; + const reply = (json: unknown, status = 200) => route.fulfill({ json, status }); + if (pathname.endsWith("/clusters")) return reply({ multiple: false, clusters: [] }); + if (pathname.endsWith("/sync")) return reply({ + ...sync(), first_ready_at: observed, total_kinds: 20, ready_kinds: timedOut() ? 19 : 20, + failing_kinds: timedOut() ? 1 : 0, unreadable_kinds: 0, + failing: timedOut() ? [{ kind: "CronJob", state: "error", class: "timeout", records: phase === "later-timeout" ? 1 : 0 }] : [], + unreadable: [], + kinds: { CronJob: { state: timedOut() ? "error" : "ready", error: timedOut() ? "timeout" : "", + records: phase === "never-listed" || phase === "empty" ? 0 : 1, + ...(phase === "never-listed" ? {} : { observed_at: observed }) } }, + }); + if (pathname.endsWith("/workloads")) { + const payload = { + items: phase === "empty" ? [] : [...resources, ...(phase === "never-listed" ? [] : [ + { resource_id: "batch~v1~cronjobs", kind: "CronJob", namespace: "shop", name: "nightly", uid: "nightly-v1" }, + ])], + counts: phase === "empty" ? { Deployment: 0, StatefulSet: 0, DaemonSet: 0, Job: 0, Pod: 0, CronJob: 0 } + : { Deployment: 2, StatefulSet: 1, DaemonSet: 0, Job: 0, Pod: 2, ...(phase === "never-listed" ? {} : { CronJob: 1 }) }, + observed_at: observed, truncated: false, sync: sync(), partial_failures: partials(), + }; + payloads.push({ phase, ...payload }); + return reply(payload); + } + if (pathname.endsWith("/namespaces")) return reply({ items: ["shop"], sync: sync() }); + if (pathname.endsWith("/overview")) return reply({ connector: "kubernetes", observed_at: observed, nodes: 1, pods: 2, namespaces: 1, workloads: 3, sync: sync() }); + if (pathname.includes("/deployment") || pathname.includes("/sso/")) return reply({ error: "Community" }, 403); + return reply({ items: [], nodes: [], edges: [], omitted: {}, sources: [], totals: {}, broken: { total: 0 }, sync: sync(), enable: false }); + }); + const panel = page.getByRole("tabpanel", { name: "Workloads", exact: true }); + const expectCount = async (kind: string, count?: number, soft = false) => { + const assertion = soft ? expect.soft : expect; + if (width === 390) await assertion(panel.getByRole("combobox", { name: "Workload kind" }).locator(`option[value="${kind}"]`)).toHaveText(count === undefined ? kind : `${kind} (${count})`, { timeout: 1000 }); + else await assertion(panel.getByRole("tab", { name: kind, exact: true })).toHaveText(count === undefined ? kind : `${kind}${count}`, { timeout: 1000 }); + }; + const shot = async () => { + await expect(panel).not.toContainText("Preparing your cluster"); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await page.screenshot({ path: capturePath("workload-count-provenance", width, "dark", phase), fullPage: true, animations: "disabled" }); + }; + await page.goto("/agent/kubernetes?view=resources"); + await expect(page.locator('[data-sync-problem="CronJob"]')).toContainText("the cluster is slow or rate-limiting reads"); + // Keep the acceptance failure while collecting independent later phases. + await expect.soft(panel).toContainText("web", { timeout: 1000 }); + await expect.soft(panel).toContainText("db", { timeout: 1000 }); + await expect.soft(panel).toContainText("Count unavailable", { timeout: 1000 }); + await expect(panel).not.toContainText("total"); + await expectCount("CronJob"); + await expectCount("All"); + await expectCount("Deployment", 2, true); + await expectCount("StatefulSet", 1, true); + await expectCount("Pod", 2, true); + await expectCount("Job", 0, true); + await shot(); + for (phase of ["known", "later-timeout"] as const) { + await page.getByRole("button", { name: "Refresh Kubernetes data", exact: true }).click(); + await expect(panel).toContainText("nightly"); + await expectCount("CronJob", 1); + await expect(panel).toContainText("6 total"); + if (phase === "later-timeout") await expect(page.locator('[data-sync-problem="CronJob"]')).toContainText("the cluster is slow or rate-limiting reads"); + await shot(); + } + phase = "empty"; + await page.getByRole("button", { name: "Refresh Kubernetes data", exact: true }).click(); + await expect(panel).toContainText("No workloads in this scope."); + await expect(panel).not.toContainText("nightly"); + await expect(panel).not.toContainText("web"); + await expect(panel).toContainText("0 total"); + await expectCount("CronJob", 0); + await expectCount("Deployment", 0); + await shot(); + await testInfo.attach("count-provenance-responses", { body: JSON.stringify(payloads), contentType: "application/json" }); + }); + } + + for (const width of [1440, 390]) for (const health of ["degraded", "failed"] as const) { + test(`pending counts and bounded post-publication ${health} recovery ${width}`, async ({ page }, testInfo) => { + await page.setViewportSize({ width, height: 900 }); + await page.addInitScript(() => localStorage.setItem("versus.theme", "dark")); + const observed = new Date().toISOString(); + let phase: "cold" | "published" | "pending" | "warming" | "empty" = "cold"; + const coldPending = ["apps~v1~deployments", "apps~v1~statefulsets", "apps~v1~daemonsets", "batch~v1~jobs", "batch~v1~cronjobs", "core~v1~pods"]; + let pending = "cronjobs", scopedReads = 0; + let release!: () => void; + const barrier = new Promise(resolve => { release = resolve; }); + const sync = () => ({ state: phase === "cold" ? "syncing" : "ready", age_s: 0, partial: false, + ...(phase === "cold" ? { pending: coldPending, pending_kinds: 20 } : phase === "pending" ? { pending: [pending] } : {}) }); + await page.route("**/api/**", async route => { + const url = new URL(route.request().url()); + const reply = (json: unknown, status = 200) => route.fulfill({ json, status }); + if (url.pathname.endsWith("/clusters")) return reply({ multiple: false, clusters: [] }); + if (url.pathname.endsWith("/sync") && phase === "cold") return reply({ + ...sync(), first_ready: false, total_kinds: 20, ready_kinds: 0, + kinds: Object.fromEntries(["Deployment", "StatefulSet", "DaemonSet", "Job", "CronJob", "Pod"].map(kind => [kind, { state: "pending", first_ready: false }])), + }); + if (url.pathname.endsWith("/sync")) return reply({ + state: health === "failed" ? "error" : "ready", partial: true, age_s: 0, first_ready_at: observed, + kinds: { Pod: { state: health === "failed" ? "error" : "ready", partial: true, observed_at: observed, records: 1, error: health === "failed" ? "unavailable" : "timeout" } }, + total_kinds: 1, ready_kinds: health === "failed" ? 0 : 1, failing_kinds: 1, pending_kinds: 0, + }); + if (url.pathname.endsWith("/workloads")) { + // The real cold response omits counts entirely; pending is not zero. + if (phase === "cold") return reply({ items: [], truncated: false, sync: sync() }); + if (phase === "warming") { + if (++scopedReads === 1) await barrier; + return reply({ code: "inventory_not_ready", error: "Scoped inventory warming", retryable: true }, 503); + } + return reply({ items: phase === "empty" ? [] : [{ resource_id: "apps~v1~deployments", kind: "Deployment", namespace: "shop", name: "committed-api", uid: "committed-uid" }], + counts: { Deployment: phase === "empty" ? 0 : 2, ...(phase === "pending" ? {} : { CronJob: 0 }) }, observed_at: observed, truncated: false, sync: sync() }); + } + if (url.pathname.endsWith("/namespaces")) return reply({ items: ["shop", "payments"], sync: sync() }); + if (url.pathname.endsWith("/overview")) return reply({ connector: "kubernetes", observed_at: observed, nodes: 1, pods: 1, namespaces: 2, workloads: 2, sync: sync() }); + if (url.pathname.includes("/deployment") || url.pathname.includes("/sso/")) return reply({ error: "Community" }, 403); + return reply({ items: [], nodes: [], edges: [], omitted: {}, sources: [], totals: {}, broken: { total: 0 }, sync: sync(), enable: false }); + }); + const panel = page.getByRole("tabpanel", { name: "Workloads", exact: true }); + const shot = async (state: string) => { + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await page.screenshot({ path: capturePath("workload-health", width, "dark", `${health}-${state}`), fullPage: true, animations: "disabled" }); + }; + try { + await page.goto("/agent/kubernetes?view=resources"); + await page.getByRole("tab", { name: "Workloads", exact: true }).click(); + await expect(panel).toContainText("Preparing your cluster"); + await expect(panel).not.toContainText("0 total"); + await expect(panel.getByRole("combobox", { name: "Workload kind" })).toHaveCount(0); + await expect(panel.getByRole("tab")).toHaveCount(0); + await shot("cold-pending"); + phase = "published"; + await page.getByRole("button", { name: "Refresh Kubernetes data", exact: true }).click(); + await expect(panel).toContainText("committed-api"); + await expect(page.locator(`[data-sync-strip="${health}"]`)).toBeVisible(); + phase = "pending"; + for (pending of ["cronjobs", "batch~v1~cronjobs"]) { + await page.getByRole("button", { name: "Refresh Kubernetes data", exact: true }).click(); + await expect(panel).toContainText("Count unavailable"); + if (width === 390) { + const kinds = panel.getByRole("combobox", { name: "Workload kind" }); + await expect(kinds.locator('option[value="CronJob"]')).toHaveText("CronJob"); + await expect(kinds.locator('option[value="Deployment"]')).toHaveText("Deployment (2)"); + await expect(kinds.locator('option[value="All"]')).toHaveText("All"); + } else { + await expect(panel.getByRole("tab", { name: "CronJob", exact: true })).toHaveText("CronJob"); + await expect(panel.getByRole("tab", { name: "Deployment", exact: true })).toContainText("2"); + await expect(panel.getByRole("tab", { name: "All", exact: true })).toHaveText("All"); + } + await expect(panel).not.toContainText("2 total"); + } + await shot("pending"); + await page.clock.install({ time: new Date() }); + phase = "warming"; + await panel.getByRole("combobox", { name: "Workload namespace", exact: true }).fill("payments"); + await page.getByRole("option", { name: "payments", exact: true }).click(); + await expect.poll(() => scopedReads).toBe(1); + await expect(panel).not.toContainText("committed-api"); + await shot("loading"); + release(); + for (let retry = 0; retry < 6; retry++) { + await page.clock.fastForward(1050); + await page.waitForTimeout(50); + } + await expect(panel.getByRole("button", { name: "Retry inventory", exact: true })).toBeVisible(); + expect(scopedReads).toBe(6); + await expect(panel).toContainText("Inventory unavailable"); + await expect(panel).not.toContainText("0 total"); + await shot("exhausted"); + await page.clock.fastForward(10_000); + expect(scopedReads).toBe(6); + phase = "empty"; + await panel.getByRole("button", { name: "Retry inventory", exact: true }).click(); + await expect(panel).toContainText("No workloads in this namespace."); + await expect(panel).toContainText("0 total"); + if (width === 390) await expect(panel.getByRole("combobox", { name: "Workload kind" }).locator('option[value="CronJob"]')).toHaveText("CronJob (0)"); + else await expect(panel.getByRole("tab", { name: "CronJob", exact: true })).toContainText("0"); + await shot("explicit-retry-empty"); + await testInfo.attach("bounded-health", { body: JSON.stringify({ classification: "fixture", health, scopedReads, automaticRetries: scopedReads - 1 }), contentType: "application/json" }); + } finally { release(); } + }); + } + + for (const width of [1440, 390]) for (const theme of ["dark", "light"] as const) { + test(`scope resync late warmup cursor recovery and denial callback fence ${width} ${theme}`, async ({ page }, testInfo) => { + test.setTimeout(120_000); + await page.setViewportSize({ width, height: 900 }); + await page.emulateMedia({ colorScheme: theme }); + await page.addInitScript(value => localStorage.setItem("versus.theme", value), theme); + const observed = new Date(Date.now() - 60_000).toISOString(); + let phase = "published", warmingReads = 0; + let releaseWarm!: () => void, releaseLate!: () => void; + const warm = new Promise(resolve => { releaseWarm = resolve; }); + const late = new Promise(resolve => { releaseLate = resolve; }); + const requests: Array<{ phase: string; path: string; query: Record }> = []; + const sync = () => ({ state: phase === "resync" || phase === "late-warmup" ? "syncing" : "ready", age_s: 0, partial: false }); + await page.route("**/api/**", async route => { + const url = new URL(route.request().url()), query = Object.fromEntries(url.searchParams); + requests.push({ phase, path: url.pathname, query }); + const reply = (json: unknown, status = 200) => route.fulfill({ json, status }); + if (url.pathname.endsWith("/clusters")) return reply({ multiple: false, clusters: [] }); + if (url.pathname.endsWith("/sync")) return phase === "denied" + ? reply({ code: "forbidden", error: "Infrastructure access denied" }, 403) + : reply({ ...sync(), first_ready_at: observed, total_kinds: 2, ready_kinds: phase === "resync" ? 1 : 2, pending_kinds: phase === "resync" ? 1 : 0, kinds: {} }); + if (url.pathname.endsWith("/workloads")) { + const namespace = query.namespace || "shop"; + const item = { resource_id: "apps~v1~deployments", kind: "Deployment", namespace, uid: `${namespace}-uid`, name: `${namespace}-committed`, ready: 1, desired: 1 }; + if (phase === "late-warmup" && namespace === "payments" && warmingReads++ === 0) { + await warm; + return reply({ code: "inventory_not_ready", error: "Scoped inventory warming", retryable: true }, 503); + } + if (phase === "hold-positive") { await late; return reply({ items: [item], counts: { Deployment: 1 }, observed_at: observed, sync: sync() }).catch(() => {}); } + if (phase === "denied") return reply({ code: "forbidden", error: "Infrastructure access denied" }, 403); + if (query.cursor) return reply({ code: "cursor_expired", error: "Inventory generation changed" }, 400); + if (phase === "resync") return reply({ code: "inventory_not_ready", error: "Replacement warming", retryable: true }, 503); + return reply({ items: phase === "empty" ? [] : [item], counts: { Deployment: phase === "empty" ? 0 : 1 }, observed_at: observed, sync: sync(), ...(phase === "cursor" ? { next: "old-generation" } : {}) }); + } + if (url.pathname.endsWith("/namespaces")) return reply({ items: ["shop", "payments"], sync: sync() }); + if (url.pathname.endsWith("/overview")) return reply({ connector: "kubernetes", observed_at: observed, nodes: 1, pods: 1, namespaces: 2, workloads: 1, sync: sync() }); + if (url.pathname.includes("/deployment") || url.pathname.includes("/sso/")) return reply({ error: "Community" }, 403); + return reply({ items: [], nodes: [], edges: [], omitted: {}, sources: [], totals: {}, broken: { total: 0 }, sync: sync(), enable: false }); + }); + const rows = page.locator(".kubernetes-workload-shell[role=listitem]"); + const panel = page.getByRole("tabpanel", { name: "Workloads", exact: true }); + const capture = async (state: string) => { + expect.soft(await page.evaluate(() => document.documentElement.scrollWidth > innerWidth)).toBe(false); + if (captureUnchanged) await page.screenshot({ path: capturePath("workload-recovery", width, theme, state), fullPage: true, animations: "disabled" }); + }; + try { + await page.goto("/agent/kubernetes"); + await page.getByRole("tab", { name: "Workloads", exact: true }).click(); + await expect(rows).toContainText("shop-committed"); + phase = "resync"; + await page.getByRole("button", { name: "Refresh Kubernetes data" }).click(); + await expect(page.locator('[data-sync-state="preparing"]')).toHaveText("Syncing"); + await expect(rows).toContainText("shop-committed"); + await expect(panel).not.toContainText("Resource inventory is unavailable."); + if (width === 390) await expect(panel.getByRole("combobox", { name: "Workload kind" }).locator('option[value="Deployment"]')).toHaveText("Deployment (1)"); + else await expect(panel.getByRole("tab", { name: "Deployment", exact: true })).toContainText("1"); + await capture("same-scope-sync"); + await page.reload(); + await page.getByRole("tab", { name: "Workloads", exact: true }).click(); + await expect(rows).toHaveCount(0); + await expect(panel).not.toContainText(/All 0|Deployment 0/); + await capture("reload-unknown"); + phase = "late-warmup"; + await panel.getByRole("combobox", { name: "Workload namespace" }).fill("payments"); + await page.getByRole("option", { name: "payments", exact: true }).click(); + await expect.poll(() => warmingReads).toBe(1); + await expect(rows).toHaveCount(0); + await capture("scope-loading"); + phase = "ready"; + // The held warmup completes after the ready status has been observed. + await page.getByRole("button", { name: "Refresh Kubernetes data" }).click(); + releaseWarm(); + await expect(rows).toContainText("payments-committed", { timeout: 45_000 }); + await expect(rows).not.toContainText("shop-committed"); + await capture("late-warmup-recovered"); + phase = "cursor"; + await page.getByRole("button", { name: "Refresh Kubernetes data" }).click(); + await panel.getByRole("button", { name: "Next page", exact: true }).click(); + await expect(panel).toContainText("This inventory page expired"); + await expect(rows).toHaveCount(0); + await capture("cursor-expired"); + await panel.getByRole("button", { name: "Restart inventory paging" }).click(); + await expect(rows).toContainText("payments-committed"); + phase = "empty"; + await page.getByRole("button", { name: "Refresh Kubernetes data" }).click(); + await expect(rows).toHaveCount(0); + await capture("complete-empty"); + phase = "ready"; + await page.getByRole("button", { name: "Refresh Kubernetes data" }).click(); + await expect(rows).toHaveCount(1); + phase = "hold-positive"; + await page.getByRole("button", { name: "Refresh Kubernetes data" }).click(); + await expect.poll(() => requests.filter(request => request.phase === "hold-positive" && request.path.endsWith("/workloads")).length).toBeGreaterThan(0); + phase = "denied"; + await page.getByRole("button", { name: "Refresh Kubernetes data" }).click(); + await expect(rows).toHaveCount(0); + releaseLate(); + await page.waitForTimeout(1000); + await expect(rows).toHaveCount(0); + await expect(panel).not.toContainText("payments-committed"); + await capture("denied-late-positive"); + } finally { + releaseWarm(); releaseLate(); + await testInfo.attach("scope-recovery-requests", { body: JSON.stringify(requests), contentType: "application/json" }); + } + }); + } + + for (const width of [1440, 390]) for (const theme of ["dark", "light"] as const) { + test(`pinned recurrence detail consumes empty continuations and explains missing capture ${width} ${theme}`, async ({ page }, testInfo) => { + await page.setViewportSize({ width, height: 1000 }); + await page.emulateMedia({ colorScheme: theme }); + await page.addInitScript(value => localStorage.setItem("versus.theme", value), theme); + const at = new Date(Date.now() - 300_000).toISOString().replace(/\.\d{3}Z$/, ".123456789Z"); + const later = new Date(Date.now() - 60_000).toISOString(); + const sync = { state: "ready", age_s: 0, partial: false }; + const root = { kind: "Deployment", namespace: "shop", name: "checkout", uid: "checkout-uid" }; + let state = "recurring"; + let denyDetail = false, holdDetail = false, heldDetail = 0; + let releaseDetail!: () => void; + const detailBarrier = new Promise(resolve => { releaseDetail = resolve; }); + const reads: Array<{ state: string; path: string; query: Record }> = []; + const selected = () => ({ root, occurrence_id: "pinned-old", state: state === "resolved" || state === "deleted" ? "resolved" : "open", rule: "pod.crash_loop", count: 1, severity: "critical", broken: false, ...(state === "missing-time" ? {} : { broke_at: at }) }); + await page.route("**/api/**", async route => { + const url = new URL(route.request().url()), query = Object.fromEntries(url.searchParams); + reads.push({ state, path: url.pathname, query }); + const reply = (json: unknown, status = 200) => route.fulfill({ json, status }); + if (url.pathname.endsWith("/clusters")) return reply({ multiple: false, clusters: [] }); + if (url.pathname.endsWith("/sync")) return reply({ ...sync, first_ready_at: at, total_kinds: 1, ready_kinds: 1, kinds: {} }); + if (url.pathname.endsWith("/issues") && denyDetail) return reply({ code: "forbidden", error: "Root access denied" }, 403); + if (url.pathname.endsWith("/issues")) return reply({ + items: [query.context === "persisted" ? { ...selected(), occurrence_id: "wrong-new", broke_at: later } : selected()], + ...(query.context === "persisted" ? { root_context: { root, occurrence: { ...selected(), occurrence_id: "wrong-new", broke_at: later }, retained: true }, selected_occurrence: selected(), selection_status: "resolved" } : {}), + totals: { critical: 1 }, broken: { total: 1 }, sync, + }); + if (url.pathname.endsWith("/timeline")) return reply({ + items: [{ type: "break", id: "pinned-old", at, issue: selected() }], + total: 1, crash_count: 0, counts: { change: 0, crash: 0, break: 1 }, gaps: [], sync, + }); + if (url.pathname.endsWith("/changes")) { + const cursor = query.cursor; + const item = (id: string) => ({ id, cluster: "fixture", ...root, type: "image_changed", fields: [{ path: "image", from: "v1", to: id }], at }); + return reply({ items: cursor === "empty-changes" ? [] : cursor ? [item("first-marker"), item("last-marker")] : [item("first-marker")], + ...(!cursor ? { next: "empty-changes" } : cursor === "empty-changes" ? { next: "last-changes" } : {}), gaps: [], sync }); + } + if (url.pathname.endsWith("/crashes")) { + const item = (id: string) => ({ id, sequence: id === "crash-first" ? 1 : 2, cluster: "fixture", namespace: "shop", pod: id, pod_uid: id, root_uid: root.uid, root_kind: root.kind, root_name: root.name, container: "api", type: "termination", reason: "OOMKilled", exit_code: 137, at, observed_at: at, time_source: "finished_at" }); + return reply({ items: query.cursor === "empty-crashes" ? [] : query.cursor ? [item("crash-first"), item("crash-last")] : [item("crash-first")], + ...(!query.cursor ? { next: "empty-crashes" } : query.cursor === "empty-crashes" ? { next: "last-crashes" } : {}), + since: query.since, until: query.until, gaps: [], capture_gaps: [], coverage: {}, limitations: [], capture: { mode: "continuous", behind: false }, sync }); + } + if (url.pathname.endsWith("/break-logs")) { + if (holdDetail) { heldDetail++; await detailBarrier; } + return reply({ + window: { from: query.since, to: query.until }, pods: [{ name: "retained-pod", uid: "retained-pod", state: "deleted", short: "retained" }], + main_container: "api", containers: [{ name: "api", main: true }], markers: [], + lines: state === "no-logs" || state === "deleted" ? [] : [{ at, pod: "retained-pod", pod_uid: "retained-pod", container: "api", init: false, slot: "previous", level: "error", text: "synthetic-filter pinned retained line", source: "captured" }], rate: [], patterns: [], sources: [], counts: { all: state === "no-logs" || state === "deleted" ? 0 : 1, errors: state === "no-logs" || state === "deleted" ? 0 : 1, warnings: 0 }, + coverage: { state: "partial", reason: "capture_not_found", tip: true, gaps: [{ from: query.since, to: query.until }] }, + }).catch(() => {}); + } + if (url.pathname.endsWith("/overview")) return reply({ connector: "kubernetes", observed_at: at, nodes: 1, pods: 1, namespaces: 1, workloads: 1, sync }); + if (url.pathname.includes("/deployment") || url.pathname.includes("/sso/")) return reply({ error: "Community" }, 403); + return reply({ items: [], nodes: [], edges: [], omitted: {}, sources: [], totals: {}, broken: { total: 0 }, sync, enable: false }); + }); + const renderWarnings: string[] = []; + page.on("console", message => { + if (message.type() === "error" && message.text().includes("Cannot update a component")) renderWarnings.push(message.text()); + }); + for (state of ["overview-entry", "issues-entry", "timeline-entry", "recurring", "resolved", "deleted", "no-logs", "missing-time"]) { + if (state.endsWith("-entry")) { + await page.goto("/agent/kubernetes"); + if (state === "timeline-entry") { + await page.getByRole("tab", { name: "Timeline", exact: true }).click(); + await page.getByRole("link", { name: "Open break details", exact: true }).click(); + } else { + if (state === "issues-entry") await page.getByRole("tab", { name: "Issues", exact: true }).click(); + await page.getByRole("button", { name: "Timeline for checkout", exact: true }).click(); + } + await expect(page.getByRole("tab", { name: "Timeline", exact: true })).toHaveAttribute("aria-selected", "true"); + expect(new URL(page.url()).searchParams.get("center")).toBe(at); + expect(new URL(page.url()).searchParams.get("occurrence_id")).toBe("pinned-old"); + } else { + await page.goto(`/agent/kubernetes?view=timeline&root_uid=${root.uid}&namespace=shop&occurrence_id=pinned-old¢er=${at}`); + await page.getByRole("tab", { name: "Timeline", exact: true }).click(); + } + const lane = page.getByRole("region", { name: "Timeline lane checkout", exact: true }); + await expect(lane).toBeVisible(); + const logs = page.getByRole("region", { name: "Logs at break", exact: true }); + await expect(logs).toBeVisible(); + await expect.poll(() => reads.filter(read => read.state === state && read.path.endsWith("/changes") && read.query.cursor === "last-changes").length).toBeGreaterThanOrEqual(1); + await expect.poll(() => reads.filter(read => read.state === state && read.path.endsWith("/crashes") && read.query.cursor === "last-crashes").length).toBeGreaterThanOrEqual(1); + expect(reads.filter(read => read.state === state && read.path.endsWith("/issues") && read.query.context === "persisted") + .every(read => read.query.center === at && read.query.occurrence_id === "pinned-old")).toBe(true); + expect.soft(await page.getByRole("button", { name: /^(Prev|Previous|Next)( page| crash page)?$/ }).count()).toBe(0); + expect.soft(await page.getByText(/^Page \d/).count()).toBe(0); + if (state === "missing-time") { + await expect(logs).toContainText(/time range.*unavailable|break time.*unresolved/i); + expect.soft(await lane.getByRole("slider").count()).toBe(0); + expect.soft(reads.filter(read => read.state === state && read.path.endsWith("/break-logs"))).toHaveLength(0); + } else { + const handle = lane.getByRole("slider", { name: "Log window start", exact: true }); + await expect(handle).toBeVisible(); + await handle.focus(); + await expect(handle).toBeFocused(); + const bracket = await handle.getAttribute("aria-valuenow"); + const chart = page.getByRole("region", { name: "Resource timeline chart", exact: true }); + const markers = chart.locator('[data-marker="change"]'); + await expect(markers).toHaveCount(2); + await markers.first().focus(); + await page.keyboard.press("Enter"); + await expect(page.getByRole("region", { name: "Timeline event details", exact: true })).toBeVisible(); + const filter = logs.getByRole("textbox", { name: "Search in log window" }); + await expect(logs.getByRole("combobox", { name: "Log container", exact: true })).toBeEnabled(); + await expect(logs.getByRole("combobox", { name: "Log container", exact: true })).not.toContainText("Loading"); + await filter.fill("synthetic-filter"); + const scrollBeforeRefresh = await chart.evaluate(element => { + element.scrollLeft = Math.min(64, element.scrollWidth - element.clientWidth); + return element.scrollLeft; + }); + if (width === 390) expect(scrollBeforeRefresh).toBeGreaterThan(0); + await page.getByRole("button", { name: "Refresh Kubernetes data" }).click(); + await expect(filter).toHaveValue("synthetic-filter"); + await expect(handle).toHaveAttribute("aria-valuenow", bracket!); + await expect(markers).toHaveCount(2); + await expect(page.getByRole("region", { name: "Timeline event details", exact: true })).toBeVisible(); + await expect.poll(() => chart.evaluate(element => element.scrollLeft)).toBe(scrollBeforeRefresh); + await chart.evaluate(element => { element.scrollLeft = 0; }); + expect.soft(reads.filter(read => read.state === state && read.path.endsWith("/break-logs")).every(read => + Date.parse(read.query.since) <= Date.parse(at) && Date.parse(read.query.until) >= Date.parse(at) && Date.parse(read.query.until) < Date.parse(later))).toBe(true); + } + expect.soft(await page.evaluate(() => document.documentElement.scrollWidth > innerWidth)).toBe(false); + if (captureUnchanged || state.endsWith("-entry")) await page.screenshot({ path: capturePath("focused-investigation", width, theme, state), fullPage: true, animations: "disabled" }); + } + try { + state = "denial-fence"; + await page.goto(`/agent/kubernetes?view=timeline&root_uid=${root.uid}&namespace=shop&occurrence_id=pinned-old¢er=${at}`); + await page.getByRole("tab", { name: "Timeline", exact: true }).click(); + await expect(page.getByRole("log", { name: "Break log lines" })).toContainText("pinned retained line"); + holdDetail = true; + await page.getByRole("region", { name: "Logs at break", exact: true }).getByRole("button", { name: "Previous container", exact: true }).click(); + await expect.poll(() => heldDetail).toBeGreaterThan(0); + denyDetail = true; + await page.getByRole("button", { name: "Refresh Kubernetes data" }).click(); + await expect(page.getByRole("slider", { name: "Log window start" })).toHaveCount(0); + releaseDetail(); + await page.waitForTimeout(1000); + await expect(page.getByRole("slider", { name: "Log window start" })).toHaveCount(0); + await expect(page.getByRole("log", { name: "Break log lines" })).toHaveCount(0); + if (captureUnchanged) await page.screenshot({ path: capturePath("focused-investigation", width, theme, "denied-late-logs"), fullPage: true, animations: "disabled" }); + } finally { releaseDetail(); } + expect(renderWarnings).toEqual([]); + fs.writeFileSync(path.join(process.env.HARNESS_RESULTS_DIR!, `focused-investigation-${width}-${theme}.json`), JSON.stringify({ classification: "fixture; deleted is retained identity without live resource, not proof of server deletion", reads }, null, 2)); + await testInfo.attach("pinned-requests", { body: JSON.stringify(reads), contentType: "application/json" }); + }); + } + + for (const width of [1440, 1024, 390]) for (const theme of ["dark", "light"] as const) { + test(`mixed workload column edges and keyboard controls ${width} ${theme}`, async ({ page }, testInfo) => { + await page.setViewportSize({ width, height: 900 }); + await page.addInitScript(value => localStorage.setItem("versus.theme", value), theme); + await page.emulateMedia({ colorScheme: theme }); + const sync = { state: "ready", age_s: 0, partial: false }; + let crowded = false; + const base = [ + { kind: "CronJob", resource_id: "batch~v1~cronjobs" }, + { kind: "Deployment", resource_id: "apps~v1~deployments" }, + { kind: "Pod", resource_id: "core~v1~pods" }, + { kind: "ConfigMap", resource_id: "core~v1~configmaps" }, + ]; + await page.route("**/api/**", route => { + const url = new URL(route.request().url()); + if (url.pathname === "/api/admin/kubernetes/workloads") { + const items = Array.from({ length: crowded ? 32 : 4 }, (_, index) => ({ + ...base[index % 4], namespace: "namespace-with-a-long-but-valid-name", + name: `${base[index % 4].kind.toLowerCase()}-${index}-${"long-name-".repeat(8)}`, + uid: `resource-${index}`, summary: { ready: 1, desired: 1 }, conditions: [], + })); + return route.fulfill({ json: { items, counts: { CronJob: crowded ? 8 : 1, Deployment: crowded ? 8 : 1, Pod: crowded ? 8 : 1, ConfigMap: crowded ? 8 : 1 }, observed_at: new Date().toISOString(), sync } }); + } + if (url.pathname.endsWith("/sync")) return route.fulfill({ json: { ...sync, first_ready_at: "2026-10-30T00:00:00Z", kinds: {}, total_kinds: 4, ready_kinds: 4 } }); + if (url.pathname.endsWith("/clusters")) return route.fulfill({ json: { multiple: false, clusters: [] } }); + if (url.pathname.endsWith("/namespaces")) return route.fulfill({ json: { items: ["namespace-with-a-long-but-valid-name"], sync } }); + if (url.pathname.endsWith("/overview")) return route.fulfill({ json: { connector: "kubernetes", nodes: 1, pods: 1, namespaces: 1, workloads: 4, observed_at: new Date().toISOString(), sync } }); + if (url.pathname.includes("/deployment") || url.pathname.includes("/sso/")) return route.fulfill({ status: 403, json: { error: "Community" } }); + return route.fulfill({ json: { items: [], sources: [], nodes: [], edges: [], omitted: {}, counts: {}, totals: {}, broken: { total: 0 }, sync, enable: false } }); + }); + await page.goto("/agent/kubernetes?view=workloads"); + await page.getByRole("tab", { name: "Workloads", exact: true }).click(); + const report = []; + for (const state of ["no-scrollbar", "scrollbar"] as const) { + if (state === "scrollbar") { + crowded = true; + await page.getByRole("button", { name: "Refresh Kubernetes data" }).click(); + } + const rows = page.locator(".kubernetes-workload-shell[role=listitem]"); + await expect(rows).toHaveCount(crowded ? 32 : 4); + const measurement = await rows.evaluateAll(elements => { + const head = document.querySelector(".kubernetes-workload-head")!; + const headers = Array.from(head.querySelector(".kubernetes-list-head")!.children); + const scroller = head.parentElement!; + return { + desktop: getComputedStyle(head).display !== "none", + scrollbar: scroller.scrollHeight > scroller.clientHeight, + gutter: scroller.offsetWidth - scroller.clientWidth, + overflow: document.documentElement.scrollWidth > innerWidth, + rows: elements.map(row => { + const button = row.querySelector(".kubernetes-list-row--workload")!; + const cells = Array.from(button.children); + return { kind: cells[2].textContent, deltas: [1, 2, 3].map(index => ({ + column: headers[index].textContent, + headerX: headers[index].getBoundingClientRect().x, + cellX: cells[index].getBoundingClientRect().x, + delta: Math.abs(headers[index].getBoundingClientRect().x - cells[index].getBoundingClientRect().x), + })), nestedButtons: button.querySelectorAll("button").length, + actionOutside: Array.from(row.querySelectorAll("button")).some(control => control.getBoundingClientRect().right > row.getBoundingClientRect().right + 1) }; + }), + }; + }); + report.push({ width, theme, state, ...measurement }); + expect.soft(measurement.scrollbar).toBe(crowded); + expect.soft(measurement.overflow).toBe(false); + if (width === 1440) expect.soft(measurement.desktop).toBe(true); + for (const row of measurement.rows) { + expect.soft(row.nestedButtons).toBe(0); + expect.soft(row.actionOutside).toBe(false); + if (measurement.desktop) for (const edge of row.deltas) expect.soft(edge.delta, `${row.kind} ${edge.column}`).toBeLessThanOrEqual(1); + } + const first = rows.first().getByRole("button").first(); + await first.focus(); + await expect(first).toBeFocused(); + const unsupported = rows.filter({ has: page.locator(".pill", { hasText: "CronJob" }) }); + expect.soft(await unsupported.getByRole("button").count()).toBe(crowded ? 8 : 1); + if (captureUnchanged || process.env.E2E_NATIVE_SCROLLBARS === "1" && width === 1440 && theme === "dark") { + await page.screenshot({ path: capturePath("workload-alignment", width, theme, state), fullPage: true, animations: "disabled" }); + } + } + fs.writeFileSync(path.join(process.env.HARNESS_RESULTS_DIR!, `workload-alignment-${width}-${theme}.json`), JSON.stringify({ classification: "fixture", thresholdCSSPixels: 1, report }, null, 2)); + await testInfo.attach("exact-column-edges", { body: JSON.stringify(report), contentType: "application/json" }); + }); + } +}); + test.describe("Kubernetes overview", () => { test.skip(process.env.E2E_KUBERNETES_BACKEND !== "fake", "requires a fresh Versus backend connected to the deterministic fake Kubernetes API"); @@ -7782,7 +8374,7 @@ for (const width of [1440, 390]) for (const theme of ["dark", "light"] as const) const capture = async (surface: string, state: string) => { expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); expect(await page.locator("html").getAttribute("data-theme")).toBe(theme === "light" ? "light" : null); - await page.screenshot({ path: capturePath(surface, width, theme, state), fullPage: true, animations: "disabled" }); + if (captureUnchanged) await page.screenshot({ path: capturePath(surface, width, theme, state), fullPage: true, animations: "disabled" }); }; await page.goto("/agent/kubernetes"); const health = page.getByRole("region", { name: "Cluster health", exact: true }); @@ -7836,7 +8428,7 @@ for (const width of [1440, 390]) for (const theme of ["dark", "light"] as const) expect(firstQuery.get("resource_uid")).toBe("checkout"); expect(firstQuery.get("limit")).toBe("100"); expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); - await page.screenshot({ path: capturePath("resource-detail-retained", width, theme, "timeline"), fullPage: true, animations: "disabled" }); + if (captureUnchanged) await page.screenshot({ path: capturePath("resource-detail-retained", width, theme, "timeline"), fullPage: true, animations: "disabled" }); await page.getByRole("button", { name: "Close panel", exact: true }).click(); await expect(health.getByRole("button", { name: "3 broken", exact: true })).toBeVisible(); phase = "warming"; @@ -7853,7 +8445,7 @@ for (const width of [1440, 390]) for (const theme of ["dark", "light"] as const) await expect(page.locator("[data-preparing-panel], [data-sync-strip]")).toHaveCount(0); expect(await page.getByRole("tablist", { name: "Kubernetes views" }).getByRole("tab").allTextContents()).toEqual(tabLabels); expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); - await page.screenshot({ path: capturePath("published-continuity", width, theme, "warming"), fullPage: true, animations: "disabled" }); + if (captureUnchanged) await page.screenshot({ path: capturePath("published-continuity", width, theme, "warming"), fullPage: true, animations: "disabled" }); expect(replies.some((reply) => reply.phase === "warming" && reply.path.endsWith("/issues") && (reply.payload as { broken?: { total: number } }).broken?.total === 0)).toBe(true); phase = "recollected"; await page.getByRole("button", { name: "Refresh Kubernetes data" }).click(); @@ -7876,7 +8468,7 @@ for (const width of [1440, 390]) for (const theme of ["dark", "light"] as const) await expect(page.locator("[data-preparing-panel]")).toHaveCount(0); await expect(page.getByText("No broken workloads", { exact: true })).toHaveCount(0); await expect(page.locator("[data-sync-strip]")).toContainText("credentials are unavailable or rejected"); - await page.screenshot({ path: capturePath("published-continuity", width, theme, "credential-failure"), fullPage: true, animations: "disabled" }); + if (captureUnchanged) await page.screenshot({ path: capturePath("published-continuity", width, theme, "credential-failure"), fullPage: true, animations: "disabled" }); await page.reload(); await expect(page.locator('[data-sync-state="failed"]')).toHaveText("Failed"); await expect(page.locator("[data-preparing-panel]")).toHaveCount(0); diff --git a/ui/tsconfig.app.tsbuildinfo b/ui/tsconfig.app.tsbuildinfo index 558ed3f4..51189ed7 100644 --- a/ui/tsconfig.app.tsbuildinfo +++ b/ui/tsconfig.app.tsbuildinfo @@ -1 +1 @@ -{"root":["./src/app.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/adminaccessnotice.tsx","./src/components/adminmemberscontrol.tsx","./src/components/agentaisettingscontrol.tsx","./src/components/agentchannelssettingscontrol.tsx","./src/components/agentmodecontrol.tsx","./src/components/agentrequiredroute.tsx","./src/components/agentssoconnectionscontrol.tsx","./src/components/alertfatiguesettingscontrol.tsx","./src/components/analysiscard.tsx","./src/components/analysisstream.tsx","./src/components/appshell.tsx","./src/components/assigndialog.tsx","./src/components/autorefreshcontrol.tsx","./src/components/bulkactionbar.tsx","./src/components/channelicon.tsx","./src/components/confirmdialog.tsx","./src/components/countsettingscontrol.tsx","./src/components/coveragemarker.tsx","./src/components/datatable.tsx","./src/components/dropdown.tsx","./src/components/enterpriselocked.tsx","./src/components/errorboundary.tsx","./src/components/filterbar.tsx","./src/components/filterpanel.tsx","./src/components/infohint.tsx","./src/components/kpitile.tsx","./src/components/kubernetesclusterprovider.tsx","./src/components/kubernetesfilterfield.tsx","./src/components/kubernetespendingnote.tsx","./src/components/kubernetespreparingpanel.tsx","./src/components/kubernetessynchealth.tsx","./src/components/kubernetessyncstateindicator.tsx","./src/components/kubernetesunreadablenote.tsx","./src/components/modal.tsx","./src/components/namespaceselect.tsx","./src/components/pageheader.tsx","./src/components/pagination.tsx","./src/components/patternbaselines.tsx","./src/components/peekpanel.tsx","./src/components/pill.tsx","./src/components/podlogviewer.tsx","./src/components/proportionalbarlist.tsx","./src/components/readinessprogress.tsx","./src/components/reportsettingscontrol.tsx","./src/components/reportsdialog.tsx","./src/components/resourcelogsbutton.tsx","./src/components/retryableerror.tsx","./src/components/runanalysisbutton.tsx","./src/components/searchinput.tsx","./src/components/segmentedcontrol.tsx","./src/components/servicecell.tsx","./src/components/servicegrouprows.tsx","./src/components/servicehealthexplorer.tsx","./src/components/servicehealthsection.tsx","./src/components/servicehealthsettingscontrol.tsx","./src/components/servicetopologystrip.tsx","./src/components/severitybadge.tsx","./src/components/shortcutoverlay.tsx","./src/components/sidebar.tsx","./src/components/skeleton.tsx","./src/components/sortheader.tsx","./src/components/sparkline.tsx","./src/components/spikesettingscontrol.tsx","./src/components/textinputmodal.tsx","./src/components/toast.tsx","./src/components/topbar.tsx","./src/components/feedback.tsx","./src/components/peekpanelfocus.ts","./src/components/shellcontext.ts","./src/components/toastcontext.ts","./src/components/chat/markdowntext.tsx","./src/components/settings/readonlyvalues.tsx","./src/components/settings/settingscard.tsx","./src/components/settings/settingslayout.tsx","./src/components/settings/sections.ts","./src/lib/agentai.ts","./src/lib/agentchannels.ts","./src/lib/alertfatiguechannel.ts","./src/lib/api.ts","./src/lib/auth.tsx","./src/lib/bulkselect.ts","./src/lib/chatstream.ts","./src/lib/clipboard.ts","./src/lib/countwindow.ts","./src/lib/format.ts","./src/lib/hooks.ts","./src/lib/incidentlist.ts","./src/lib/incidentpresentation.ts","./src/lib/kubernetesconnector.ts","./src/lib/kubernetescontinuity.ts","./src/lib/kubernetesevidence.ts","./src/lib/kubernetesfirstsync.ts","./src/lib/kubernetesissueprompt.ts","./src/lib/kuberneteslogtargets.ts","./src/lib/kubernetespending.ts","./src/lib/kubernetesprojection.ts","./src/lib/kubernetesresourcedetail.ts","./src/lib/kubernetesresourcepresentation.ts","./src/lib/kubernetessynchealth.ts","./src/lib/kubernetestime.ts","./src/lib/lazymodule.ts","./src/lib/learnexclude.ts","./src/lib/listfilters.ts","./src/lib/localadmin.ts","./src/lib/markdownpolicy.ts","./src/lib/pagination.ts","./src/lib/readiness.ts","./src/lib/reportanalytics.ts","./src/lib/reportschedule.ts","./src/lib/role.ts","./src/lib/rowactions.ts","./src/lib/servicegrace.ts","./src/lib/servicegroups.ts","./src/lib/servicehealthpresentation.ts","./src/lib/serviceoverride.ts","./src/lib/servicetopologypresentation.ts","./src/lib/severity.ts","./src/lib/signinsurface.ts","./src/lib/sloadvisor.ts","./src/lib/sortrows.ts","./src/lib/spikerows.ts","./src/lib/sse.ts","./src/lib/ssoconfig.ts","./src/lib/ssoreturn.ts","./src/lib/systempromptnav.ts","./src/lib/theme.ts","./src/lib/useautorefresh.ts","./src/lib/usebulkselection.ts","./src/lib/usedeploymentorg.ts","./src/lib/useeffectiverole.ts","./src/lib/usefacetfilters.ts","./src/lib/usekubernetesbreaklogs.ts","./src/lib/usekubernetescluster.tsx","./src/lib/uselearnexclusions.ts","./src/lib/usepodlogstream.ts","./src/lib/useservicegrouping.ts","./src/lib/useservicehealth.ts","./src/lib/useserviceintelquery.ts","./src/lib/usestickybottom.ts","./src/pages/adminpage.tsx","./src/pages/agentconfigpage.tsx","./src/pages/agentoverviewpage.tsx","./src/pages/agenttoolspage.tsx","./src/pages/alertfatiguepage.tsx","./src/pages/analyseslistpage.tsx","./src/pages/analysisdetailpage.tsx","./src/pages/chatpage.tsx","./src/pages/decisionspage.tsx","./src/pages/detectdetailpage.tsx","./src/pages/incidentdetailpage.tsx","./src/pages/incidentsconfigpage.tsx","./src/pages/incidentspage.tsx","./src/pages/kubernetesagenttab.tsx","./src/pages/kubernetesbreaklogs.tsx","./src/pages/kubernetesbreaktimeline.tsx","./src/pages/kubernetesexplorerviews.tsx","./src/pages/kubernetesfleet.tsx","./src/pages/kubernetesissuealerts.tsx","./src/pages/kuberneteslinkedsource.tsx","./src/pages/kubernetespage.tsx","./src/pages/kubernetesresourcedetail.tsx","./src/pages/kubernetesresourcemetrics.tsx","./src/pages/kuberneteswarnings.tsx","./src/pages/learnedsignalsview.tsx","./src/pages/memberspage.tsx","./src/pages/nowpage.tsx","./src/pages/patterndetailpage.tsx","./src/pages/patternspage.tsx","./src/pages/peoplepage.tsx","./src/pages/runbookspage.tsx","./src/pages/slorecommendationspage.tsx","./src/pages/servicedetailpage.tsx","./src/pages/servicespage.tsx","./src/pages/settingspage.tsx","./src/pages/shadowdetailpage.tsx","./src/pages/systempromptpage.tsx","./src/pages/teamspage.tsx","./src/pages/kubernetesexplorertabs.ts","./src/pages/kuberneteswarningitems.ts","./src/pages/kubernetes-resource-detail/events.tsx","./src/pages/kubernetes-resource-detail/identity.tsx","./src/pages/kubernetes-resource-detail/relations.tsx","./src/pages/kubernetes-resource-detail/timeline.tsx","./src/pages/kubernetes-resource-detail/yaml.tsx"],"version":"5.9.3"} \ No newline at end of file +{"root":["./src/app.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/adminaccessnotice.tsx","./src/components/adminmemberscontrol.tsx","./src/components/agentaisettingscontrol.tsx","./src/components/agentchannelssettingscontrol.tsx","./src/components/agentmodecontrol.tsx","./src/components/agentrequiredroute.tsx","./src/components/agentssoconnectionscontrol.tsx","./src/components/alertfatiguesettingscontrol.tsx","./src/components/analysiscard.tsx","./src/components/analysisstream.tsx","./src/components/appshell.tsx","./src/components/assigndialog.tsx","./src/components/autorefreshcontrol.tsx","./src/components/bulkactionbar.tsx","./src/components/channelicon.tsx","./src/components/confirmdialog.tsx","./src/components/countsettingscontrol.tsx","./src/components/coveragemarker.tsx","./src/components/datatable.tsx","./src/components/dropdown.tsx","./src/components/enterpriselocked.tsx","./src/components/errorboundary.tsx","./src/components/filterbar.tsx","./src/components/filterpanel.tsx","./src/components/infohint.tsx","./src/components/kpitile.tsx","./src/components/kubernetesclusterprovider.tsx","./src/components/kubernetesfilterfield.tsx","./src/components/kubernetespendingnote.tsx","./src/components/kubernetespreparingpanel.tsx","./src/components/kubernetessynchealth.tsx","./src/components/kubernetessyncstateindicator.tsx","./src/components/kubernetesunreadablenote.tsx","./src/components/modal.tsx","./src/components/namespaceselect.tsx","./src/components/pageheader.tsx","./src/components/pagination.tsx","./src/components/patternbaselines.tsx","./src/components/peekpanel.tsx","./src/components/pill.tsx","./src/components/podlogviewer.tsx","./src/components/proportionalbarlist.tsx","./src/components/readinessprogress.tsx","./src/components/reportsettingscontrol.tsx","./src/components/reportsdialog.tsx","./src/components/resourcelogsbutton.tsx","./src/components/retryableerror.tsx","./src/components/runanalysisbutton.tsx","./src/components/searchinput.tsx","./src/components/segmentedcontrol.tsx","./src/components/servicecell.tsx","./src/components/servicegrouprows.tsx","./src/components/servicehealthexplorer.tsx","./src/components/servicehealthsection.tsx","./src/components/servicehealthsettingscontrol.tsx","./src/components/servicetopologystrip.tsx","./src/components/severitybadge.tsx","./src/components/shortcutoverlay.tsx","./src/components/sidebar.tsx","./src/components/skeleton.tsx","./src/components/sortheader.tsx","./src/components/sparkline.tsx","./src/components/spikesettingscontrol.tsx","./src/components/textinputmodal.tsx","./src/components/toast.tsx","./src/components/topbar.tsx","./src/components/feedback.tsx","./src/components/peekpanelfocus.ts","./src/components/shellcontext.ts","./src/components/toastcontext.ts","./src/components/chat/markdowntext.tsx","./src/components/settings/readonlyvalues.tsx","./src/components/settings/settingscard.tsx","./src/components/settings/settingslayout.tsx","./src/components/settings/sections.ts","./src/lib/agentai.ts","./src/lib/agentchannels.ts","./src/lib/alertfatiguechannel.ts","./src/lib/api.ts","./src/lib/auth.tsx","./src/lib/bulkselect.ts","./src/lib/chatstream.ts","./src/lib/clipboard.ts","./src/lib/countwindow.ts","./src/lib/format.ts","./src/lib/hooks.ts","./src/lib/incidentlist.ts","./src/lib/incidentpresentation.ts","./src/lib/kubernetesconnector.ts","./src/lib/kubernetescontinuity.ts","./src/lib/kubernetesevidence.ts","./src/lib/kubernetesfirstsync.ts","./src/lib/kubernetesfocusedhistory.ts","./src/lib/kubernetesissueprompt.ts","./src/lib/kuberneteslogtargets.ts","./src/lib/kubernetespending.ts","./src/lib/kubernetesprojection.ts","./src/lib/kubernetesresourcedetail.ts","./src/lib/kubernetesresourcepresentation.ts","./src/lib/kubernetessynchealth.ts","./src/lib/kubernetestime.ts","./src/lib/lazymodule.ts","./src/lib/learnexclude.ts","./src/lib/listfilters.ts","./src/lib/localadmin.ts","./src/lib/markdownpolicy.ts","./src/lib/pagination.ts","./src/lib/readiness.ts","./src/lib/reportanalytics.ts","./src/lib/reportschedule.ts","./src/lib/role.ts","./src/lib/rowactions.ts","./src/lib/servicegrace.ts","./src/lib/servicegroups.ts","./src/lib/servicehealthpresentation.ts","./src/lib/serviceoverride.ts","./src/lib/servicetopologypresentation.ts","./src/lib/severity.ts","./src/lib/signinsurface.ts","./src/lib/sloadvisor.ts","./src/lib/sortrows.ts","./src/lib/spikerows.ts","./src/lib/sse.ts","./src/lib/ssoconfig.ts","./src/lib/ssoreturn.ts","./src/lib/systempromptnav.ts","./src/lib/theme.ts","./src/lib/useautorefresh.ts","./src/lib/usebulkselection.ts","./src/lib/usedeploymentorg.ts","./src/lib/useeffectiverole.ts","./src/lib/usefacetfilters.ts","./src/lib/usekubernetesbreaklogs.ts","./src/lib/usekubernetescluster.tsx","./src/lib/uselearnexclusions.ts","./src/lib/usepodlogstream.ts","./src/lib/useservicegrouping.ts","./src/lib/useservicehealth.ts","./src/lib/useserviceintelquery.ts","./src/lib/usestickybottom.ts","./src/pages/adminpage.tsx","./src/pages/agentconfigpage.tsx","./src/pages/agentoverviewpage.tsx","./src/pages/agenttoolspage.tsx","./src/pages/alertfatiguepage.tsx","./src/pages/analyseslistpage.tsx","./src/pages/analysisdetailpage.tsx","./src/pages/chatpage.tsx","./src/pages/decisionspage.tsx","./src/pages/detectdetailpage.tsx","./src/pages/incidentdetailpage.tsx","./src/pages/incidentsconfigpage.tsx","./src/pages/incidentspage.tsx","./src/pages/kubernetesagenttab.tsx","./src/pages/kubernetesbreaklogs.tsx","./src/pages/kubernetesbreaktimeline.tsx","./src/pages/kubernetesexplorerviews.tsx","./src/pages/kubernetesfleet.tsx","./src/pages/kubernetesissuealerts.tsx","./src/pages/kuberneteslinkedsource.tsx","./src/pages/kubernetespage.tsx","./src/pages/kubernetesresourcedetail.tsx","./src/pages/kubernetesresourcemetrics.tsx","./src/pages/kuberneteswarnings.tsx","./src/pages/learnedsignalsview.tsx","./src/pages/memberspage.tsx","./src/pages/nowpage.tsx","./src/pages/patterndetailpage.tsx","./src/pages/patternspage.tsx","./src/pages/peoplepage.tsx","./src/pages/runbookspage.tsx","./src/pages/slorecommendationspage.tsx","./src/pages/servicedetailpage.tsx","./src/pages/servicespage.tsx","./src/pages/settingspage.tsx","./src/pages/shadowdetailpage.tsx","./src/pages/systempromptpage.tsx","./src/pages/teamspage.tsx","./src/pages/kubernetesexplorertabs.ts","./src/pages/kuberneteswarningitems.ts","./src/pages/kubernetes-resource-detail/events.tsx","./src/pages/kubernetes-resource-detail/identity.tsx","./src/pages/kubernetes-resource-detail/relations.tsx","./src/pages/kubernetes-resource-detail/timeline.tsx","./src/pages/kubernetes-resource-detail/yaml.tsx"],"version":"5.9.3"} \ No newline at end of file