diff --git a/internal/daemon/reconciler.go b/internal/daemon/reconciler.go index 7a873336..78db2011 100644 --- a/internal/daemon/reconciler.go +++ b/internal/daemon/reconciler.go @@ -204,6 +204,20 @@ func (r *BootcNodeReconciler) reconcileBootcNode( }, nil } + // The cached status can be stale — for example a read taken while + // the node was still settling into the image it just rebooted + // into. A booted/desired mismatch leads to destructive action + // (stage, reboot), so re-verify it with a live bootc status read + // before acting, and keep using the fresh view below. + if !image.InfoMatchesDigest(bn.Status.Booted, digested.Digest().String()) { + if fresh, err := r.StatusWatcher.RefreshStatus(ctx); err != nil { + log.Info("Live bootc status refresh failed, continuing with cached status", + "error", err) + } else { + applyBootcStatus(bn, fresh) + } + } + // Nothing to do the desired image matches the booted ones. // Reset the stage backoff to start from a clean state. if image.InfoMatchesDigest(bn.Status.Booted, digested.Digest().String()) { @@ -370,11 +384,17 @@ func (r *BootcNodeReconciler) populateBootcFields( return fmt.Errorf("getting bootc status: %w", err) } + applyBootcStatus(bn, status) + + return nil +} + +// applyBootcStatus copies the bootc status entries onto the BootcNode +// status fields. +func applyBootcStatus(bn *bootcv1alpha1.BootcNode, status *bootc.Status) { bn.Status.Booted = convertBootEntry(status.Status.Booted) bn.Status.Staged = convertBootEntry(status.Status.Staged) bn.Status.Rollback = convertBootEntry(status.Status.Rollback) - - return nil } // updateAction represents the next step the daemon should take for an diff --git a/internal/daemon/reconciler_test.go b/internal/daemon/reconciler_test.go index 3ea02aec..dc9d555c 100644 --- a/internal/daemon/reconciler_test.go +++ b/internal/daemon/reconciler_test.go @@ -12,6 +12,7 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" "github.com/bootc-dev/bootc-operator/internal/bootc" @@ -383,6 +384,58 @@ func TestRollback(t *testing.T) { g.Expect(fake.getRebooted()).To(BeFalse()) } +// TestStaleStatusNoSpuriousRestage verifies that a booted/desired +// mismatch in the watcher's cached status is re-verified with a live +// bootc status read before the daemon stages or reboots. A stale +// cache — for example a read taken while the node was still settling +// into the image it just rebooted into — must not cause the daemon to +// re-stage and re-reboot an image the node already booted. +func TestStaleStatusNoSpuriousRestage(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + // Post-reboot reality: the node is booted on image C. The executor + // is what a live bootc status read reports. + fake := &fakeExecutor{} + fake.status = newBootcStatus(testutil.DigestC) + + // Stale cache: still believes the node is booted on image A. + stale := newBootcStatus(testutil.DigestA) + watcher := &StatusWatcher{ + Events: make(chan event.GenericEvent, 1), + NodeName: testNodeName, + Executor: fake, + } + watcher.cached = &stale + watcher.started = true + + r := &BootcNodeReconciler{ + NodeName: testNodeName, + Executor: fake, + StatusWatcher: watcher, + stageDone: make(chan event.GenericEvent, 1), + } + t.Cleanup(func() { + r.inflight.mu.Lock() + if r.inflight.cancel != nil { + r.inflight.cancel() + } + r.inflight.mu.Unlock() + }) + + // The desired image is C, which matches what is actually booted. + bn := testutil.NewNode(testNodeName, testutil.ImageDigestRefC) + + res, err := r.reconcileBootcNode(ctx, bn) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res.needsReboot).To(BeFalse(), + "must not reboot a node already booted on the desired image") + g.Expect(fake.getStageImg()).To(BeEmpty(), + "stale cache must not trigger a spurious re-stage of the already-booted image") + g.Expect(bn.Status.Booted.ImageDigest).To(Equal(testutil.DigestC), + "the live re-read must correct the stale booted status") +} + func TestCancelInflightStage(t *testing.T) { g := NewWithT(t) g.SetDefaultEventuallyTimeout(pollTimeout) diff --git a/internal/daemon/watcher.go b/internal/daemon/watcher.go index 1bfa49ad..bfb01252 100644 --- a/internal/daemon/watcher.go +++ b/internal/daemon/watcher.go @@ -53,6 +53,14 @@ func (w *StatusWatcher) GetStatus(ctx context.Context) (*bootc.Status, error) { return s, err } +// RefreshStatus reads bootc status live from the host, bypassing and +// updating the cache, and returns the fresh status. Callers use it to +// re-verify the cached view before acting on it. +func (w *StatusWatcher) RefreshStatus(ctx context.Context) (*bootc.Status, error) { + s, _, err := w.refresh(ctx) + return s, err +} + // refresh reads bootc status from the host and updates the cache. // Returns the new status, if the cache has changed or an error. func (w *StatusWatcher) refresh(ctx context.Context) (*bootc.Status, bool, error) {