Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 22 additions & 2 deletions internal/daemon/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,20 @@ func (r *BootcNodeReconciler) reconcileBootcNode(
}, nil
}

// The cached status can be stale — for example a read taken while
// the node was still settling into the image it just rebooted
// into. A booted/desired mismatch leads to destructive action
// (stage, reboot), so re-verify it with a live bootc status read
// before acting, and keep using the fresh view below.
if !image.InfoMatchesDigest(bn.Status.Booted, digested.Digest().String()) {
if fresh, err := r.StatusWatcher.RefreshStatus(ctx); err != nil {
log.Info("Live bootc status refresh failed, continuing with cached status",
"error", err)
} else {
applyBootcStatus(bn, fresh)
}
}

// Nothing to do the desired image matches the booted ones.
// Reset the stage backoff to start from a clean state.
if image.InfoMatchesDigest(bn.Status.Booted, digested.Digest().String()) {
Expand Down Expand Up @@ -370,11 +384,17 @@ func (r *BootcNodeReconciler) populateBootcFields(
return fmt.Errorf("getting bootc status: %w", err)
}

applyBootcStatus(bn, status)

return nil
}

// applyBootcStatus copies the bootc status entries onto the BootcNode
// status fields.
func applyBootcStatus(bn *bootcv1alpha1.BootcNode, status *bootc.Status) {
bn.Status.Booted = convertBootEntry(status.Status.Booted)
bn.Status.Staged = convertBootEntry(status.Status.Staged)
bn.Status.Rollback = convertBootEntry(status.Status.Rollback)

return nil
}

// updateAction represents the next step the daemon should take for an
Expand Down
53 changes: 53 additions & 0 deletions internal/daemon/reconciler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
. "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"

bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1"
"github.com/bootc-dev/bootc-operator/internal/bootc"
Expand Down Expand Up @@ -383,6 +384,58 @@ func TestRollback(t *testing.T) {
g.Expect(fake.getRebooted()).To(BeFalse())
}

// TestStaleStatusNoSpuriousRestage verifies that a booted/desired
// mismatch in the watcher's cached status is re-verified with a live
// bootc status read before the daemon stages or reboots. A stale
// cache — for example a read taken while the node was still settling
// into the image it just rebooted into — must not cause the daemon to
// re-stage and re-reboot an image the node already booted.
func TestStaleStatusNoSpuriousRestage(t *testing.T) {
g := NewWithT(t)
ctx := context.Background()

// Post-reboot reality: the node is booted on image C. The executor
// is what a live bootc status read reports.
fake := &fakeExecutor{}
fake.status = newBootcStatus(testutil.DigestC)

// Stale cache: still believes the node is booted on image A.
stale := newBootcStatus(testutil.DigestA)
watcher := &StatusWatcher{
Events: make(chan event.GenericEvent, 1),
NodeName: testNodeName,
Executor: fake,
}
watcher.cached = &stale
watcher.started = true

r := &BootcNodeReconciler{
NodeName: testNodeName,
Executor: fake,
StatusWatcher: watcher,
stageDone: make(chan event.GenericEvent, 1),
}
t.Cleanup(func() {
r.inflight.mu.Lock()
if r.inflight.cancel != nil {
r.inflight.cancel()
}
r.inflight.mu.Unlock()
})

// The desired image is C, which matches what is actually booted.
bn := testutil.NewNode(testNodeName, testutil.ImageDigestRefC)

res, err := r.reconcileBootcNode(ctx, bn)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res.needsReboot).To(BeFalse(),
"must not reboot a node already booted on the desired image")
g.Expect(fake.getStageImg()).To(BeEmpty(),
"stale cache must not trigger a spurious re-stage of the already-booted image")
g.Expect(bn.Status.Booted.ImageDigest).To(Equal(testutil.DigestC),
"the live re-read must correct the stale booted status")
}

func TestCancelInflightStage(t *testing.T) {
g := NewWithT(t)
g.SetDefaultEventuallyTimeout(pollTimeout)
Expand Down
8 changes: 8 additions & 0 deletions internal/daemon/watcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,14 @@ func (w *StatusWatcher) GetStatus(ctx context.Context) (*bootc.Status, error) {
return s, err
}

// RefreshStatus reads bootc status live from the host, bypassing and
// updating the cache, and returns the fresh status. Callers use it to
// re-verify the cached view before acting on it.
func (w *StatusWatcher) RefreshStatus(ctx context.Context) (*bootc.Status, error) {
s, _, err := w.refresh(ctx)
return s, err
}

// refresh reads bootc status from the host and updates the cache.
// Returns the new status, if the cache has changed or an error.
func (w *StatusWatcher) refresh(ctx context.Context) (*bootc.Status, bool, error) {
Expand Down
Loading