From f97fbb15f5ccb062d0f8c40cc83841c0a09c4400 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Wed, 26 Aug 2026 21:58:15 -0400 Subject: [PATCH] Decode whole words in bulk in bitmapContainer's many-iterator nextMany walked set bits one at a time. When the current word is exhausted it now decodes whole words with the AVX-512 kernel for as long as the caller's buffer has room for the 64 values a single word could produce, falling back to the word-at-a-time loop for the partial word. bitset stays zero across the bulk step, preserving the invariant that word `base` has been fully consumed. The word count per step is room/64, which cannot overflow the buffer whatever the data looks like. Xeon Gold 6548N, ns per value, 4096-entry buffer: cardinality 4096 1.240 -> 0.418 (2.97x) cardinality 16384 1.348 -> 0.130 (10.4x) cardinality 32768 1.229 -> 0.097 (12.7x) cardinality 65536 1.021 -> 0.074 (13.8x) BenchmarkRealDataNextMany over the whole corpus: census-income 10.19 ms -> 3.41 ms (2.99x) weather_sept_85 20.22 ms -> 7.42 ms (2.72x) A 64-entry buffer is the worst case, one word per bulk step; it is still slightly faster (1.21x), not a regression. nextMany64 is left alone: []uint64 output needs a byte-to-quadword widen, a different kernel with a different trade-off. Claude-Session: https://claude.ai/code/session_0123ePBefqPjrCvhxdwWFakj --- bitmapcontainer.go | 23 ++++++++++- manyiterator_bitmap_test.go | 79 +++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 manyiterator_bitmap_test.go diff --git a/bitmapcontainer.go b/bitmapcontainer.go index 354af17e..91a50a73 100644 --- a/bitmapcontainer.go +++ b/bitmapcontainer.go @@ -175,16 +175,35 @@ func (bcmi *bitmapContainerManyIterator) nextMany(hs uint32, buf []uint32) int { n := 0 base := bcmi.base bitset := bcmi.bitset + bitmap := bcmi.ptr.bitmap for n < len(buf) { if bitset == 0 { + // The current word is exhausted, so whole words follow. Decode them + // in bulk while the buffer still has room for the 64 values a single + // word could produce; anything left over falls through to the loop + // below. bitset stays zero, which keeps the iterator's invariant + // that word `base` has been fully consumed. + for useVectorFill { + words := (len(buf) - n) / 64 + if remaining := len(bitmap) - base - 1; words > remaining { + words = remaining + } + if words <= 0 { + break + } + start := base + 1 + n = fillLeastSignificant16bitsVector(bitmap[start:start+words], buf, n, hs|uint32(start)*64) + base += words + } + base++ - if base >= len(bcmi.ptr.bitmap) { + if base >= len(bitmap) { bcmi.base = base bcmi.bitset = bitset return n } - bitset = bcmi.ptr.bitmap[base] + bitset = bitmap[base] continue } t := bitset & -bitset diff --git a/manyiterator_bitmap_test.go b/manyiterator_bitmap_test.go new file mode 100644 index 00000000..da3ae938 --- /dev/null +++ b/manyiterator_bitmap_test.go @@ -0,0 +1,79 @@ +package roaring + +import ( + "math/bits" + "math/rand" + "testing" +) + +// bitmapContainerManyIterator decodes whole words in bulk when the buffer has +// room and falls back to the word-at-a-time loop otherwise. Buffer sizes that +// are not multiples of 64 make the two paths interleave. +func TestBitmapContainerNextManyBufferSizes(t *testing.T) { + r := rand.New(rand.NewSource(11)) + for _, cardinality := range []int{1, 63, 64, 65, 4096, 30000, 65535, 65536} { + bc := newBitmapContainer() + for bc.cardinality < cardinality { + bc.iadd(uint16(r.Intn(65536))) + } + + want := make([]uint32, 0, cardinality) + for k, w := range bc.bitmap { + for w != 0 { + want = append(want, 0x50000|uint32(k*64+bits.TrailingZeros64(w))) + w &= w - 1 + } + } + + for _, bufSize := range []int{1, 2, 63, 64, 65, 100, 127, 128, 129, 1000, 4096, 65536} { + it := bc.getManyIterator() + buf := make([]uint32, bufSize) + got := make([]uint32, 0, cardinality) + for { + n := it.nextMany(0x50000, buf) + if n == 0 { + break + } + if n > bufSize { + t.Fatalf("card=%d buf=%d: returned %d, more than the buffer holds", cardinality, bufSize, n) + } + got = append(got, buf[:n]...) + } + if len(got) != len(want) { + t.Fatalf("card=%d buf=%d: got %d values, want %d", cardinality, bufSize, len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("card=%d buf=%d: at %d got %d want %d", cardinality, bufSize, i, got[i], want[i]) + } + } + } + } +} + +func benchBitmapNextMany(b *testing.B, cardinality, bufSize int) { + r := rand.New(rand.NewSource(12)) + bc := newBitmapContainer() + for bc.cardinality < cardinality { + bc.iadd(uint16(r.Intn(65536))) + } + buf := make([]uint32, bufSize) + var sink int + for b.Loop() { + it := bc.getManyIterator() + for n := it.nextMany(0x50000, buf); n != 0; n = it.nextMany(0x50000, buf) { + sink += n + } + } + _ = sink + b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N*cardinality), "ns/value") +} + +func BenchmarkBitmapContainerNextMany(b *testing.B) { + b.Run("card=4096/buf=4096", func(b *testing.B) { benchBitmapNextMany(b, 4096, 4096) }) + b.Run("card=16384/buf=4096", func(b *testing.B) { benchBitmapNextMany(b, 16384, 4096) }) + b.Run("card=32768/buf=4096", func(b *testing.B) { benchBitmapNextMany(b, 32768, 4096) }) + b.Run("card=65536/buf=4096", func(b *testing.B) { benchBitmapNextMany(b, 65536, 4096) }) + b.Run("card=32768/buf=512", func(b *testing.B) { benchBitmapNextMany(b, 32768, 512) }) + b.Run("card=32768/buf=64", func(b *testing.B) { benchBitmapNextMany(b, 32768, 64) }) +}