From dbe56df1f2568cf886bc390239b1ceb6ec874b9b Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Thu, 3 Sep 2026 13:06:52 -0700
Subject: [PATCH] Fix flaky compactor tests: ignore user index update loop
 errors

userIndexUpdateLoop runs in the background of every sharding-enabled
compactor test and ticks every UsersScanner.UpdateInterval, which
prepare() sets to 100ms. Each tick calls ownUser() against the ring, so
any tick landing before the lifecycler is ACTIVE, or after it starts
LEAVING, logs an error.

The tests that assert the compactor's complete log output with
ElementsMatch then fail with "elements differ" depending purely on how
many ticks fell inside the test window. This showed up as
TestCompactor_ShouldCompactAllUsersOnShardingEnabledButOnlyOneInstanceRunning
failing twice in a row on test (arm64), with a different number of extra
log lines each time.

removeIgnoredLogs already suppresses the other two lines this loop can
emit, but not these two, which carry a variable err= payload and so need
a regex rather than an exact match.

Add a unit test for the helper, since the flake itself is timing
dependent and does not reproduce reliably.

Fixes #7826

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 pkg/compactor/compactor_test.go | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/pkg/compactor/compactor_test.go b/pkg/compactor/compactor_test.go
index c50a5dc695b..64c9d72df12 100644
--- a/pkg/compactor/compactor_test.go
+++ b/pkg/compactor/compactor_test.go
@@ -1522,6 +1522,12 @@ func removeIgnoredLogs(input []string) []string {
 	ignoredLogStringsRegexList := []*regexp.Regexp{
 		regexp.MustCompile(`^level=(info|debug|warn) component=cleaner .+$`),
 		regexp.MustCompile(`^level=info component=compactor msg="set state" .+$`),
+		// The user index update loop ticks every UsersScanner.UpdateInterval (100ms in tests) and
+		// logs these whenever it runs while the ring has no healthy instance, which happens before
+		// the lifecycler is ACTIVE and again once it starts LEAVING. Both carry a variable err=
+		// payload, so they can't be matched exactly like the other lines this loop emits.
+		regexp.MustCompile(`^level=error component=compactor msg="failed to check if compactor owns updating user index" err=.+$`),
+		regexp.MustCompile(`^level=error component=compactor msg="failed to update user index" err=.+$`),
 	}
 
 	out := make([]string, 0, len(input))
@@ -1558,6 +1564,27 @@ main:
 	return out
 }
 
+func TestRemoveIgnoredLogs_UserIndexUpdateLoop(t *testing.T) {
+	t.Parallel()
+
+	// userIndexUpdateLoop runs in the background of every sharding-enabled compactor test and ticks
+	// every UsersScanner.UpdateInterval (100ms in tests), so any line it can emit must be ignored.
+	// Otherwise it non-deterministically leaks into the tests that assert the exact log output.
+	ignored := []string{
+		`level=error component=compactor msg="failed to check if compactor owns updating user index" err="at least 1 live replicas required, could only find 0 - unhealthy instances: 1.2.3.4:0"`,
+		`level=error component=compactor msg="failed to update user index" err="mocked error"`,
+		`level=error component=compactor msg="context timeout, exit user index update loop" err="context canceled"`,
+		`level=info component=compactor msg="successfully updated user index" duration_ms=1`,
+	}
+	assert.Empty(t, removeIgnoredLogs(ignored))
+
+	// Unrelated compactor errors must still come through.
+	kept := []string{
+		`level=error component=compactor msg="failed to compact user blocks" user=user-1 err="mocked error"`,
+	}
+	assert.Equal(t, kept, removeIgnoredLogs(kept))
+}
+
 func prepareConfig() Config {
 	compactorCfg := Config{}
 	flagext.DefaultValues(&compactorCfg)