From 22c2c3dd180ac235ffb4cc93321dfb7f13180e50 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 25 Aug 2026 01:21:42 +0200 Subject: [PATCH] vendor: github.com/bits-and-blooms/bitset v1.25.0 removes fallbacks for legacy Go versions full diff: https://github.com/bits-and-blooms/bitset/compare/v1.13.0...v1.25.0 Signed-off-by: Sebastiaan van Stijn --- go.mod | 2 +- go.sum | 4 +- .../github.com/bits-and-blooms/bitset/LICENSE | 2 +- .../bits-and-blooms/bitset/README.md | 89 +- .../bits-and-blooms/bitset/bitset.go | 857 +- .../bits-and-blooms/bitset/bitset_iter.go | 31 + .../bits-and-blooms/bitset/pext.gen.go | 8866 +++++++++++++++++ .../bits-and-blooms/bitset/popcnt.go | 90 +- .../bits-and-blooms/bitset/popcnt_19.go | 62 - .../bits-and-blooms/bitset/popcnt_amd64.go | 68 - .../bits-and-blooms/bitset/popcnt_amd64.s | 104 - .../bits-and-blooms/bitset/popcnt_generic.go | 25 - .../bits-and-blooms/bitset/run_fuzz_tests.sh | 40 + .../bits-and-blooms/bitset/select.go | 8 +- .../bitset/trailing_zeros_18.go | 15 - .../bitset/trailing_zeros_19.go | 10 - vendor/modules.txt | 2 +- 17 files changed, 9827 insertions(+), 448 deletions(-) create mode 100644 vendor/github.com/bits-and-blooms/bitset/bitset_iter.go create mode 100644 vendor/github.com/bits-and-blooms/bitset/pext.gen.go delete mode 100644 vendor/github.com/bits-and-blooms/bitset/popcnt_19.go delete mode 100644 vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.go delete mode 100644 vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.s delete mode 100644 vendor/github.com/bits-and-blooms/bitset/popcnt_generic.go create mode 100644 vendor/github.com/bits-and-blooms/bitset/run_fuzz_tests.sh delete mode 100644 vendor/github.com/bits-and-blooms/bitset/trailing_zeros_18.go delete mode 100644 vendor/github.com/bits-and-blooms/bitset/trailing_zeros_19.go diff --git a/go.mod b/go.mod index b21cf45ac2..2c11d48754 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.25.0 require ( code.cloudfoundry.org/clock v1.1.0 github.com/Microsoft/go-winio v0.6.2 - github.com/bits-and-blooms/bitset v1.13.0 + github.com/bits-and-blooms/bitset v1.25.0 github.com/cloudflare/cfssl v1.6.4 github.com/container-storage-interface/spec v1.2.0 github.com/distribution/reference v0.5.0 diff --git a/go.sum b/go.sum index a7c6944d04..d953e89c5b 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= -github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.25.0 h1:0Ro0qF4abCkM6SqWPVj29sFhAbMPAZpaDD7xhJ10beM= +github.com/bits-and-blooms/bitset v1.25.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= diff --git a/vendor/github.com/bits-and-blooms/bitset/LICENSE b/vendor/github.com/bits-and-blooms/bitset/LICENSE index 59cab8a939..72ee212ed4 100644 --- a/vendor/github.com/bits-and-blooms/bitset/LICENSE +++ b/vendor/github.com/bits-and-blooms/bitset/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2014 Will Fitzgerald. All rights reserved. +Copyright (c) 2014 the bitset authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/vendor/github.com/bits-and-blooms/bitset/README.md b/vendor/github.com/bits-and-blooms/bitset/README.md index fe7bca65eb..0317a242a6 100644 --- a/vendor/github.com/bits-and-blooms/bitset/README.md +++ b/vendor/github.com/bits-and-blooms/bitset/README.md @@ -2,20 +2,11 @@ *Go language library to map between non-negative integers and boolean values* -[![Test](https://github.com/bits-and-blooms/bitset/workflows/Test/badge.svg)](https://github.com/willf/bitset/actions?query=workflow%3ATest) -[![Go Report Card](https://goreportcard.com/badge/github.com/willf/bitset)](https://goreportcard.com/report/github.com/willf/bitset) +[![Test](https://github.com/bits-and-blooms/bitset/workflows/Test/badge.svg)](https://github.com/bits-and-blooms/bitset/actions?query=workflow%3ATest) [![PkgGoDev](https://pkg.go.dev/badge/github.com/bits-and-blooms/bitset?tab=doc)](https://pkg.go.dev/github.com/bits-and-blooms/bitset?tab=doc) -This library is part of the [awesome go collection](https://github.com/avelino/awesome-go). It is used in production by several important systems: - -* [beego](https://github.com/beego/beego) -* [CubeFS](https://github.com/cubefs/cubefs) -* [Amazon EKS Distro](https://github.com/aws/eks-distro) -* [sourcegraph](https://github.com/sourcegraph/sourcegraph) -* [torrent](https://github.com/anacrolix/torrent) - - +This library is part of the [awesome go collection](https://github.com/avelino/awesome-go). It is used in production by several important systems. ## Description Package bitset implements bitsets, a mapping between non-negative integers and boolean values. @@ -25,7 +16,7 @@ It provides methods for setting, clearing, flipping, and testing individual inte But it also provides set intersection, union, difference, complement, and symmetric operations, as well as tests to check whether any, all, or no bits are set, and querying a bitset's current length and number of positive bits. -BitSets are expanded to the size of the largest set bit; the memory allocation is approximately Max bits, where Max is the largest set bit. BitSets are never shrunk. On creation, a hint can be given for the number of bits that will be used. +BitSets are expanded to the size of the largest set bit; the memory allocation is approximately Max bits, where Max is the largest set bit. BitSets are never shrunk automatically, but `Shrink` and `Compact` methods are available. On creation, a hint can be given for the number of bits that will be used. Many of the methods, including Set, Clear, and Flip, return a BitSet pointer, which allows for chaining. @@ -69,6 +60,13 @@ func main() { } ``` +If you have Go 1.23 or better, you can iterate over the set bits like so: + +```go +for i := range b.EachSet() {} +``` + + Package documentation is at: https://pkg.go.dev/github.com/bits-and-blooms/bitset?tab=doc @@ -125,7 +123,7 @@ E.g., ## Memory Usage -The memory usage of a bitset using `N` bits is at least `N/8` bytes. The number of bits in a bitset is at least as large as one plus the greatest bit index you have accessed. Thus it is possible to run out of memory while using a bitset. If you have lots of bits, you might prefer compressed bitsets, like the [Roaring bitmaps](http://roaringbitmap.org) and its [Go implementation](https://github.com/RoaringBitmap/roaring). +The memory usage of a bitset using `N` bits is at least `N/8` bytes. The number of bits in a bitset is at least as large as one plus the greatest bit index you have accessed. Thus it is possible to run out of memory while using a bitset. If you have lots of bits, you might prefer compressed bitsets, like the [Roaring bitmaps](https://roaringbitmap.org) and its [Go implementation](https://github.com/RoaringBitmap/roaring). The `roaring` library allows you to go back and forth between compressed Roaring bitmaps and the conventional bitset instances: ```Go @@ -134,11 +132,11 @@ The `roaring` library allows you to go back and forth between compressed Roaring ``` -## Implementation Note +### Goroutine safety -Go 1.9 introduced a native `math/bits` library. We provide backward compatibility to Go 1.7, which might be removed. +In general, it's not safe to access the same BitSet using different goroutines--they are unsynchronized for performance. -It is possible that a later version will match the `math/bits` return signature for counts (which is `int`, rather than our library's `uint64`). If so, the version will be bumped. +Should you want to access a BitSet from more than one goroutine, you should provide synchronization. Typically this is done by using channels to pass the *BitSet around (in Go style; so there is only ever one owner), or by using `sync.Mutex` to serialize operations on BitSets. ## Installation @@ -146,6 +144,59 @@ It is possible that a later version will match the `math/bits` return signature go get github.com/bits-and-blooms/bitset ``` +## Users + + + +Databases & Storage +- [milvus-io/milvus](https://github.com/milvus-io/milvus) — popular open-source vector database +- [cubefs/cubefs](https://github.com/cubefs/cubefs) — CNCF distributed filesystem (used at JD.com) +- [sourcenetwork/defradb](https://github.com/sourcenetwork/defradb) — decentralized document database +- [siglens/siglens](https://github.com/siglens/siglens) — open-source observability/log analytics platform +- [ByteStorage/FlyDB](https://github.com/ByteStorage/FlyDB) — high-performance KV store + +Search & Indexing +- [blevesearch/bleve](https://github.com/blevesearch/bleve) — full-text search library for Go +- [blevesearch/vellum](https://github.com/blevesearch/vellum) — FST library used internally by Bleve +- [blugelabs/bluge](https://github.com/blugelabs/bluge) — modern Go search library (successor to Bleve) +- [geange/lucene-go](https://github.com/geange/lucene-go) — Lucene port to Go + +Blockchain & Cryptography +- [consensys/gnark](https://github.com/consensys/gnark) — zk-SNARK proving system +- [consensys/gnark-crypto](https://github.com/consensys/gnark-crypto) — elliptic curve cryptography for ZK proofs +- [NethermindEth/juno](https://github.com/NethermindEth/juno) — StarkNet full node +- [onflow/cadence](https://github.com/onflow/cadence) — Flow blockchain smart contract language runtime +- [cosmos/cosmos-sdk](https://github.com/cosmos/cosmos-sdk) — Cosmos blockchain SDK (slashing module) +- [babylonlabs-io/babylon](https://github.com/babylonlabs-io/babylon) — Bitcoin staking protocol +- [bnb-chain/greenfield](https://github.com/bnb-chain/greenfield) — BNB Chain decentralized storage + +Infrastructure & Networking +- [moby/swarmkit](https://github.com/moby/swarmkit) — Docker Swarm orchestration (ID management) +- [coredhcp/coredhcp](https://github.com/coredhcp/coredhcp) — pluggable DHCP server (bitmap IP allocation) +- [Scalingo/sand](https://github.com/Scalingo/sand) — private networking for containers (IP allocation) +- [AliyunContainerService/terway](https://github.com/AliyunContainerService/terway) — Alibaba Cloud K8s CNI +- [pomerium/pomerium](https://github.com/pomerium/pomerium) — identity-aware access proxy + +Distributed Systems & Messaging +- [apache/pulsar-client-go](https://github.com/apache/pulsar-client-go) — official Apache Pulsar Go client +- [d7y.io/dragonfly](https://github.com/dragonflyoss/Dragonfly2) — CNCF P2P file distribution system +- [RoaringBitmap/roaring](https://github.com/RoaringBitmap/roaring) — compressed bitmap library (interops with bitset) +- [zhenghaoz/gorse](https://github.com/zhenghaoz/gorse) — Go recommendation engine + +Machine Learning / Vector Search +- [fogfish/hnsw](https://github.com/fogfish/hnsw) — HNSW approximate nearest-neighbor graph +- [semafind/semadb](https://github.com/semafind/semadb) — vector database + +Developer Tools & Security +- [bearer/bearer](https://github.com/bearer/bearer) — security code scanner (AST traversal) +- [zrepl/zrepl](https://github.com/zrepl/zrepl) — ZFS replication tool +- [NVIDIA/dcgm-exporter](https://github.com/NVIDIA/dcgm-exporter) — NVIDIA GPU Prometheus exporter +- [NVIDIA/go-dcgm](https://github.com/NVIDIA/go-dcgm) — NVIDIA datacenter GPU management bindings +- [gohugoio/hugo](https://github.com/gohugoio/hugo) — the Hugo static site generator +- [godoctor/godoctor](https://github.com/godoctor/godoctor) — Go refactoring tool (dataflow analysis) + + + ## Contributing If you wish to contribute to this project, please branch and issue a pull request against master ("[GitHub Flow](https://guides.github.com/introduction/flow/)") @@ -157,3 +208,9 @@ Before committing the code, please check if it passes tests, has adequate covera go test go test -cover ``` + + +## Further reading + +

Mastering Programming: From Testing to Performance in Go

+
diff --git a/vendor/github.com/bits-and-blooms/bitset/bitset.go b/vendor/github.com/bits-and-blooms/bitset/bitset.go index 9f38ed3a9d..3cfa6001cc 100644 --- a/vendor/github.com/bits-and-blooms/bitset/bitset.go +++ b/vendor/github.com/bits-and-blooms/bitset/bitset.go @@ -13,7 +13,8 @@ bitset's current length and number of positive bits. BitSets are expanded to the size of the largest set bit; the memory allocation is approximately Max bits, where Max is -the largest set bit. BitSets are never shrunk. On creation, +the largest set bit. BitSets are never shrunk automatically, but +the Shrink and Compact methods are available. On creation, a hint can be given for the number of bits that will be used. Many of the methods, including Set,Clear, and Flip, return @@ -21,13 +22,13 @@ a BitSet pointer, which allows for chaining. Example use: - import "bitset" - var b BitSet + import "github.com/bits-and-blooms/bitset" + var b bitset.BitSet b.Set(10).Set(11) if b.Test(1000) { b.Clear(1000) } - if B.Intersection(bitset.New(100).Set(10)).Count() > 1 { + if b.Intersection(bitset.New(100).Set(10)).Count() == 1 { fmt.Println("Intersection works.") } @@ -44,17 +45,21 @@ import ( "errors" "fmt" "io" + "math/bits" "strconv" ) // the wordSize of a bit set -const wordSize = uint(64) +const wordSize = 64 // the wordSize of a bit set in bytes const wordBytes = wordSize / 8 +// wordMask is wordSize-1, used for bit indexing in a word +const wordMask = wordSize - 1 + // log2WordSize is lg(wordSize) -const log2WordSize = uint(6) +const log2WordSize = 6 // allBits has every bit set const allBits uint64 = 0xffffffffffffffff @@ -68,9 +73,16 @@ var base64Encoding = base64.URLEncoding // Base64StdEncoding Marshal/Unmarshal BitSet with base64.StdEncoding(Default: base64.URLEncoding) func Base64StdEncoding() { base64Encoding = base64.StdEncoding } -// LittleEndian Marshal/Unmarshal Binary as Little Endian(Default: binary.BigEndian) +// LittleEndian sets Marshal/Unmarshal Binary as Little Endian (Default: binary.BigEndian) func LittleEndian() { binaryOrder = binary.LittleEndian } +// BigEndian sets Marshal/Unmarshal Binary as Big Endian (Default: binary.BigEndian) +func BigEndian() { binaryOrder = binary.BigEndian } + +// BinaryOrder returns the current binary order, see also LittleEndian() +// and BigEndian() to change the order. +func BinaryOrder() binary.ByteOrder { return binaryOrder } + // A BitSet is a set of bits. The zero value of a BitSet is an empty set of length 0. type BitSet struct { length uint @@ -88,7 +100,7 @@ func (b *BitSet) safeSet() []uint64 { return b.set } -// SetBitsetFrom fills the bitset with an array of integers without creating a new BitSet instance +// SetBitsetFrom fills the bitset with an array of words without creating a new BitSet instance. func (b *BitSet) SetBitsetFrom(buf []uint64) { b.length = uint(len(buf)) * 64 b.set = buf @@ -99,36 +111,58 @@ func From(buf []uint64) *BitSet { return FromWithLength(uint(len(buf))*64, buf) } -// FromWithLength constructs from an array of words and length. -func FromWithLength(len uint, set []uint64) *BitSet { - return &BitSet{len, set} +// FromWithLength constructs from an array of words and length in bits. +// This function is for advanced users, most users should prefer +// the From function. +// As a user of FromWithLength, you are responsible for ensuring +// that the length is correct: your slice should have length at +// least (length+63)/64 in 64-bit words. +func FromWithLength(length uint, set []uint64) *BitSet { + if len(set) < wordsNeeded(length) { + panic("BitSet.FromWithLength: slice is too short") + } + return &BitSet{length, set} } -// Bytes returns the bitset as array of words +// Bytes returns the bitset as array of 64-bit words, giving direct access to the internal representation. +// It is not a copy, so changes to the returned slice will affect the bitset. +// It is meant for advanced users. +// +// Deprecated: Bytes is deprecated. Use [BitSet.Words] instead. func (b *BitSet) Bytes() []uint64 { return b.set } +// Words returns the bitset as array of 64-bit words, giving direct access to the internal representation. +// It is not a copy, so changes to the returned slice will affect the bitset. +// It is meant for advanced users. +func (b *BitSet) Words() []uint64 { + return b.set +} + // wordsNeeded calculates the number of words needed for i bits func wordsNeeded(i uint) int { - if i > (Cap() - wordSize + 1) { + if i > (Cap() - wordMask) { return int(Cap() >> log2WordSize) } - return int((i + (wordSize - 1)) >> log2WordSize) + return int((i + wordMask) >> log2WordSize) } // wordsNeededUnbound calculates the number of words needed for i bits, possibly exceeding the capacity. -// This function is useful if you know that the capacity cannot be exceeded (e.g., you have an existing bitmap). +// This function is useful if you know that the capacity cannot be exceeded (e.g., you have an existing BitSet). func wordsNeededUnbound(i uint) int { - return int((i + (wordSize - 1)) >> log2WordSize) + return (int(i) + wordMask) >> log2WordSize } // wordsIndex calculates the index of words in a `uint64` func wordsIndex(i uint) uint { - return i & (wordSize - 1) + return i & wordMask } -// New creates a new BitSet with a hint that length bits will be required +// New creates a new BitSet with a hint that length bits will be required. +// The memory usage is at least length/8 bytes. +// In case of allocation failure, the function will return a BitSet with zero +// capacity. func New(length uint) (bset *BitSet) { defer func() { if r := recover(); r != nil { @@ -147,13 +181,30 @@ func New(length uint) (bset *BitSet) { return bset } +// MustNew creates a new BitSet with the given length bits. +// It panics if length exceeds the possible capacity or by a lack of memory. +func MustNew(length uint) (bset *BitSet) { + if length >= Cap() { + panic("You are exceeding the capacity") + } + + return &BitSet{ + length, + make([]uint64, wordsNeeded(length)), // may panic on lack of memory + } +} + // Cap returns the total possible capacity, or number of bits +// that can be stored in the BitSet theoretically. Under 32-bit system, +// it is 4294967295 and under 64-bit system, it is 18446744073709551615. +// Note that this is further limited by the maximum allocation size in Go, +// and your available memory, as any Go data structure. func Cap() uint { return ^uint(0) } // Len returns the number of bits in the BitSet. -// Note the difference to method Count, see example. +// Note that it differs from the Count function. func (b *BitSet) Len() uint { return b.length } @@ -184,12 +235,32 @@ func (b *BitSet) Test(i uint) bool { return b.set[i>>log2WordSize]&(1<> log2WordSize) + subWordIndex := wordsIndex(i) + + // The word that the index falls within, shifted so the index is at bit 0 + var firstWord, secondWord uint64 + if firstWordIndex < len(b.set) { + firstWord = b.set[firstWordIndex] >> subWordIndex + } + + // The next word, masked to only include the necessary bits and shifted to cover the + // top of the word + if (firstWordIndex + 1) < len(b.set) { + secondWord = b.set[firstWordIndex+1] << uint64(wordSize-subWordIndex) + } + + return firstWord | secondWord +} + // Set bit i to 1, the capacity of the bitset is automatically // increased accordingly. -// If i>= Cap(), this function will panic. // Warning: using a very large value for 'i' // may lead to a memory shortage and a panic: the caller is responsible // for providing sensible parameters in line with their memory capacity. +// The memory usage is at least slightly over i/8 bytes. func (b *BitSet) Set(i uint) *BitSet { if i >= b.length { // if we need more bits, make 'em b.extendSet(i) @@ -198,7 +269,7 @@ func (b *BitSet) Set(i uint) *BitSet { return b } -// Clear bit i to 0 +// Clear bit i to 0. This never causes a memory allocation. It is always safe. func (b *BitSet) Clear(i uint) *BitSet { if i >= b.length { return b @@ -208,7 +279,6 @@ func (b *BitSet) Clear(i uint) *BitSet { } // SetTo sets bit i to value. -// If i>= Cap(), this function will panic. // Warning: using a very large value for 'i' // may lead to a memory shortage and a panic: the caller is responsible // for providing sensible parameters in line with their memory capacity. @@ -219,8 +289,49 @@ func (b *BitSet) SetTo(i uint, value bool) *BitSet { return b.Clear(i) } +// SetRange sets bits in [start, end) to 1, the capacity of the bitset is +// automatically increased accordingly. +// Warning: using a very large value for 'end' +// may lead to a memory shortage and a panic: the caller is responsible +// for providing sensible parameters in line with their memory capacity. +func (b *BitSet) SetRange(start, end uint) *BitSet { + if start >= end { + return b + } + + if end-1 >= b.length { + b.extendSet(end - 1) + } + + startWord := start >> log2WordSize + endWord := (end - 1) >> log2WordSize // inclusive, the word holding bit end-1 + + // e.g. start = 71 -> wordsIndex(start) = 7 + // firstMask = ^uint64(0) << 7 = 0b111111....11110000000 + // keeps the bits below start untouched + firstMask := ^uint64(0) << wordsIndex(start) + + // e.g. end = 135 -> wordsIndex(-end) = 57, see FlipRange for the + // modular arithmetic of the unary minus + // lastMask = ^uint64(0) >> 57 = 0b00000....0001111111 + // keeps the bits from end on untouched + lastMask := ^uint64(0) >> wordsIndex(-end) + + if startWord == endWord { // the whole range lives in a single word + b.set[startWord] |= firstMask & lastMask + return b + } + + b.set[startWord] |= firstMask + for i := startWord + 1; i < endWord; i++ { + b.set[i] = ^uint64(0) + } + b.set[endWord] |= lastMask + + return b +} + // Flip bit at i. -// If i>= Cap(), this function will panic. // Warning: using a very large value for 'i' // may lead to a memory shortage and a panic: the caller is responsible // for providing sensible parameters in line with their memory capacity. @@ -233,7 +344,6 @@ func (b *BitSet) Flip(i uint) *BitSet { } // FlipRange bit in [start, end). -// If end>= Cap(), this function will panic. // Warning: using a very large value for 'end' // may lead to a memory shortage and a panic: the caller is responsible // for providing sensible parameters in line with their memory capacity. @@ -241,23 +351,54 @@ func (b *BitSet) FlipRange(start, end uint) *BitSet { if start >= end { return b } + if end-1 >= b.length { // if we need more bits, make 'em b.extendSet(end - 1) } - var startWord uint = start >> log2WordSize - var endWord uint = end >> log2WordSize + + startWord := int(start >> log2WordSize) + endWord := int(end >> log2WordSize) + + // b.set[startWord] ^= ^(^uint64(0) << wordsIndex(start)) + // e.g: + // start = 71, + // startWord = 1 + // wordsIndex(start) = 71 % 64 = 7 + // (^uint64(0) << 7) = 0b111111....11110000000 + // + // mask = ^(^uint64(0) << 7) = 0b000000....00001111111 + // + // flips the first 7 bits in b.set[1] and + // in the range loop, the b.set[1] gets again flipped + // so the two expressions flip results in a flip + // in b.set[1] from [7,63] + // + // handle startWord special, get's reflipped in range loop b.set[startWord] ^= ^(^uint64(0) << wordsIndex(start)) - if endWord > 0 { - // bounds check elimination - data := b.set - _ = data[endWord-1] - for i := startWord; i < endWord; i++ { - data[i] = ^data[i] - } - } - if end&(wordSize-1) != 0 { - b.set[endWord] ^= ^uint64(0) >> wordsIndex(-end) + + for idx := range b.set[startWord:endWord] { + b.set[startWord+idx] = ^b.set[startWord+idx] + } + + // handle endWord special + // e.g. + // end = 135 + // endWord = 2 + // + // wordsIndex(-7) = 57 + // see the golang spec: + // "For unsigned integer values, the operations +, -, *, and << are computed + // modulo 2n, where n is the bit width of the unsigned integer's type." + // + // mask = ^uint64(0) >> 57 = 0b00000....0001111111 + // + // flips in b.set[2] from [0,7] + // + // is end at word boundary? + if idx := wordsIndex(-end); idx != 0 { + b.set[endWord] ^= ^uint64(0) >> wordsIndex(idx) } + return b } @@ -275,6 +416,7 @@ func (b *BitSet) FlipRange(start, end uint) *BitSet { // memory usage until the GC runs. Normally this should not be a problem, but if you // have an extremely large BitSet its important to understand that the old BitSet will // remain in memory until the GC frees it. +// If you are memory constrained, this function may cause a panic. func (b *BitSet) Shrink(lastbitindex uint) *BitSet { length := lastbitindex + 1 idx := wordsNeeded(length) @@ -294,6 +436,11 @@ func (b *BitSet) Shrink(lastbitindex uint) *BitSet { // Compact shrinks BitSet to so that we preserve all set bits, while minimizing // memory usage. Compact calls Shrink. +// A new slice is allocated to store the new bits, so you may see an increase in +// memory usage until the GC runs. Normally this should not be a problem, but if you +// have an extremely large BitSet its important to understand that the old BitSet will +// remain in memory until the GC frees it. +// If you are memory constrained, this function may cause a panic. func (b *BitSet) Compact() *BitSet { idx := len(b.set) - 1 for ; idx >= 0 && b.set[idx] == 0; idx-- { @@ -353,7 +500,8 @@ func (b *BitSet) InsertAt(idx uint) *BitSet { return b } -// String creates a string representation of the Bitmap +// String creates a string representation of the BitSet. It is only intended for +// human-readable output and not for serialization. func (b *BitSet) String() string { // follows code from https://github.com/RoaringBitmap/roaring var buffer bytes.Buffer @@ -412,9 +560,67 @@ func (b *BitSet) DeleteAt(i uint) *BitSet { b.length = b.length - 1 + // the bitset may now use one word less: shrink the slice so that + // len(b.set) keeps matching the number of words in use, as the rest + // of the package expects. Otherwise, functions that scan the whole + // slice (e.g., SetAll, Count) would operate on a word that lies + // beyond the length of the bitset. + if wordCount := b.wordCount(); wordCount < len(b.set) { + // the discarded words must be zeroed: extendSet may later revive + // them with a fast resize, and they must not carry stale bits. + for i := wordCount; i < len(b.set); i++ { + b.set[i] = 0 + } + b.set = b.set[:wordCount] + } + return b } +// AppendTo appends all set bits to buf and returns the (maybe extended) buf. +// In case of allocation failure, the function will panic. +// +// See also [BitSet.AsSlice] and [BitSet.NextSetMany]. +func (b *BitSet) AppendTo(buf []uint) []uint { + // In theory, we could overflow uint, but in practice, we will not. + for idx, word := range b.set { + for word != 0 { + // In theory idx<= len(b.set) { return 0, false } - w := b.set[x] - w = w >> wordsIndex(i) - if w != 0 { - return i + trailingZeroes64(w), true + + // process first (partial) word + word := b.set[x] >> wordsIndex(i) + if word != 0 { + return i + uint(bits.TrailingZeros64(word)), true } + + // process the following full words until next bit is set + // x < len(b.set), no out-of-bounds panic in following slice expression x++ - // bounds check elimination in the loop - if x < 0 { - return 0, false - } - for x < len(b.set) { - if b.set[x] != 0 { - return uint(x)*wordSize + trailingZeroes64(b.set[x]), true + for idx, word := range b.set[x:] { + if word != 0 { + return uint((x+idx)<> log2WordSize) if x >= len(b.set) || capacity == 0 { - return 0, myanswer[:0] + return 0, result[:0] } - skip := wordsIndex(i) - word := b.set[x] >> skip - myanswer = myanswer[:capacity] - size := int(0) + + // process first (partial) word + word := b.set[x] >> wordsIndex(i) + + size := 0 for word != 0 { - r := trailingZeroes64(word) - t := word & ((^word) + 1) - myanswer[size] = r + i + result[size] = i + uint(bits.TrailingZeros64(word)) + size++ if size == capacity { - goto End + return result[size-1], result[:size] } - word = word ^ t + + // clear the rightmost set bit + word &= word - 1 } + + // process the following full words + // x < len(b.set), no out-of-bounds panic in following slice expression x++ for idx, word := range b.set[x:] { for word != 0 { - r := trailingZeroes64(word) - t := word & ((^word) + 1) - myanswer[size] = r + (uint(x+idx) << 6) + result[size] = uint((x+idx)< 0 { - return myanswer[size-1], myanswer[:size] + return result[size-1], result[:size] } - return 0, myanswer[:0] + return 0, result[:0] } // NextClear returns the next clear bit from the specified index, // including possibly the current index // along with an error code (true = valid, false = no bit found i.e. all bits are set) func (b *BitSet) NextClear(i uint) (uint, bool) { + if i >= b.length { + return 0, false + } x := int(i >> log2WordSize) if x >= len(b.set) { return 0, false } - w := b.set[x] - w = w >> wordsIndex(i) - wA := allBits >> wordsIndex(i) - index := i + trailingZeroes64(^w) - if w != wA && index < b.length { + + // process first (maybe partial) word + word := b.set[x] + word = word >> wordsIndex(i) + wordAll := allBits >> wordsIndex(i) + + index := i + uint(bits.TrailingZeros64(^word)) + if word != wordAll && index < b.length { return index, true } + + // process the following full words until next bit is cleared + // x < len(b.set), no out-of-bounds panic in following slice expression x++ - // bounds check elimination in the loop - if x < 0 { - return 0, false - } - for x < len(b.set) { - if b.set[x] != allBits { - index = uint(x)*wordSize + trailingZeroes64(^b.set[x]) + for idx, word := range b.set[x:] { + if word != allBits { + index = uint((x+idx)<> log2WordSize) + if x >= len(b.set) { + return 0, false + } + word := b.set[x] + + // Clear the bits above the index + word = word & ((1 << (wordsIndex(i) + 1)) - 1) + if word != 0 { + return uint(x<= 0; x-- { + word = b.set[x] + if word != 0 { + return uint(x<> log2WordSize) + if x >= len(b.set) { + return 0, false + } + word := b.set[x] + + // Flip all bits and find the highest one bit + word = ^word + + // Clear the bits above the index + word = word & ((1 << (wordsIndex(i) + 1)) - 1) + + if word != 0 { + return uint(x<= 0; x-- { + word = b.set[x] + word = ^word + if word != 0 { + return uint(x< 0 { + cnt += popcntMaskSlice(b.set[:l], compare.set[:l]) + } cnt += popcntSlice(b.set[l:]) return uint(cnt) } @@ -726,6 +1006,7 @@ func sortByLength(a *BitSet, b *BitSet) (ap *BitSet, bp *BitSet) { // Intersection of base set and other set // This is the BitSet equivalent of & (and) +// In case of allocation failure, the function will return an empty BitSet. func (b *BitSet) Intersection(compare *BitSet) (result *BitSet) { panicIfNull(b) panicIfNull(compare) @@ -737,10 +1018,13 @@ func (b *BitSet) Intersection(compare *BitSet) (result *BitSet) { return } -// IntersectionCardinality computes the cardinality of the union +// IntersectionCardinality computes the cardinality of the intersection func (b *BitSet) IntersectionCardinality(compare *BitSet) uint { panicIfNull(b) panicIfNull(compare) + if b.length == 0 || compare.length == 0 { + return 0 + } b, compare = sortByLength(b, compare) cnt := popcntAndSlice(b.set, compare.set) return uint(cnt) @@ -791,13 +1075,16 @@ func (b *BitSet) Union(compare *BitSet) (result *BitSet) { return } -// UnionCardinality computes the cardinality of the uniton of the base set +// UnionCardinality computes the cardinality of the union of the base set // and the compare set. func (b *BitSet) UnionCardinality(compare *BitSet) uint { panicIfNull(b) panicIfNull(compare) b, compare = sortByLength(b, compare) - cnt := popcntOrSlice(b.set, compare.set) + cnt := uint64(0) + if len(b.set) > 0 { + cnt += popcntOrSlice(b.set, compare.set) + } if len(compare.set) > len(b.set) { cnt += popcntSlice(compare.set[len(b.set):]) } @@ -852,7 +1139,10 @@ func (b *BitSet) SymmetricDifferenceCardinality(compare *BitSet) uint { panicIfNull(b) panicIfNull(compare) b, compare = sortByLength(b, compare) - cnt := popcntXorSlice(b.set, compare.set) + cnt := uint64(0) + if len(b.set) > 0 { + cnt += popcntXorSlice(b.set, compare.set) + } if len(compare.set) > len(b.set) { cnt += popcntSlice(compare.set[len(b.set):]) } @@ -900,6 +1190,7 @@ func (b *BitSet) cleanLastWord() { } // Complement computes the (local) complement of a bitset (up to length bits) +// In case of allocation failure, the function will return an empty BitSet. func (b *BitSet) Complement() (result *BitSet) { panicIfNull(b) result = New(b.length) @@ -958,6 +1249,7 @@ func (b *BitSet) IsStrictSuperSet(other *BitSet) bool { // DumpAsBits dumps a bit set as a string of bits. Following the usual convention in Go, // the least significant bits are printed last (index 0 is at the end of the string). +// This is useful for debugging and testing. It is not suitable for serialization. func (b *BitSet) DumpAsBits() string { if b.set == nil { return "." @@ -972,18 +1264,18 @@ func (b *BitSet) DumpAsBits() string { // BinaryStorageSize returns the binary storage requirements (see WriteTo) in bytes. func (b *BitSet) BinaryStorageSize() int { - return int(wordBytes + wordBytes*uint(b.wordCount())) + return wordBytes + wordBytes*b.wordCount() } func readUint64Array(reader io.Reader, data []uint64) error { length := len(data) bufferSize := 128 - buffer := make([]byte, bufferSize*int(wordBytes)) + buffer := make([]byte, bufferSize*wordBytes) for i := 0; i < length; i += bufferSize { end := i + bufferSize if end > length { end = length - buffer = buffer[:wordBytes*uint(end-i)] + buffer = buffer[:wordBytes*(end-i)] } chunk := data[i:end] if _, err := io.ReadFull(reader, buffer); err != nil { @@ -998,12 +1290,12 @@ func readUint64Array(reader io.Reader, data []uint64) error { func writeUint64Array(writer io.Writer, data []uint64) error { bufferSize := 128 - buffer := make([]byte, bufferSize*int(wordBytes)) + buffer := make([]byte, bufferSize*wordBytes) for i := 0; i < len(data); i += bufferSize { end := i + bufferSize if end > len(data) { end = len(data) - buffer = buffer[:wordBytes*uint(end-i)] + buffer = buffer[:wordBytes*(end-i)] } chunk := data[i:end] for i, x := range chunk { @@ -1020,6 +1312,15 @@ func writeUint64Array(writer io.Writer, data []uint64) error { // WriteTo writes a BitSet to a stream. The format is: // 1. uint64 length // 2. []uint64 set +// The length is the number of bits in the BitSet. +// +// The set is a slice of uint64s containing between length and length + 63 bits. +// It is interpreted as a big-endian array of uint64s by default (see BinaryOrder()) +// meaning that the first 8 bits are stored at byte index 7, the next 8 bits are stored +// at byte index 6... the bits 64 to 71 are stored at byte index 8, etc. +// If you change the binary order, you need to do so for both reading and writing. +// We recommend using the default binary order. +// // Upon success, the number of bytes written is returned. // // Performance: if this function is used to write to a disk or network @@ -1050,6 +1351,7 @@ func (b *BitSet) WriteTo(stream io.Writer) (int64, error) { // The format is: // 1. uint64 length // 2. []uint64 set +// See WriteTo for details. // Upon success, the number of bytes read is returned. // If the current BitSet is not large enough to hold the data, // it is extended. In case of error, the BitSet is either @@ -1101,6 +1403,7 @@ func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { } // MarshalBinary encodes a BitSet into a binary form and returns the result. +// Please see WriteTo for details. func (b *BitSet) MarshalBinary() ([]byte, error) { var buf bytes.Buffer _, err := b.WriteTo(&buf) @@ -1112,6 +1415,7 @@ func (b *BitSet) MarshalBinary() ([]byte, error) { } // UnmarshalBinary decodes the binary form generated by MarshalBinary. +// Please see WriteTo for details. func (b *BitSet) UnmarshalBinary(data []byte) error { buf := bytes.NewReader(data) _, err := b.ReadFrom(buf) @@ -1149,19 +1453,39 @@ func (b *BitSet) UnmarshalJSON(data []byte) error { return err } -// Rank returns the nunber of set bits up to and including the index +// Rank returns the number of set bits up to and including the index // that are set in the bitset. // See https://en.wikipedia.org/wiki/Ranking#Ranking_in_statistics -func (b *BitSet) Rank(index uint) uint { - if index >= b.length { - return b.Count() +func (b *BitSet) Rank(index uint) (rank uint) { + index++ // Rank is up to and including + + // needed more than once + length := len(b.set) + + // TODO: built-in min requires go1.21 or later + // idx := min(int(index>>6), len(b.set)) + idx := int(index >> 6) + if idx > length { + idx = length } - leftover := (index + 1) & 63 - answer := uint(popcntSlice(b.set[:(index+1)>>6])) - if leftover != 0 { - answer += uint(popcount(b.set[(index+1)>>6] << (64 - leftover))) + + // sum up the popcounts until idx ... + // TODO: cannot range over idx (...): requires go1.22 or later + // for j := range idx { + for j := 0; j < idx; j++ { + if w := b.set[j]; w != 0 { + rank += uint(bits.OnesCount64(w)) + } + } + + // ... plus partial word at idx, + // make Rank inlineable and faster in the end + // don't test index&63 != 0, just add, less branching + if idx < length { + rank += uint(bits.OnesCount64(b.set[idx] << (64 - index&63))) } - return answer + + return } // Select returns the index of the jth set bit, where j is the argument. @@ -1174,7 +1498,7 @@ func (b *BitSet) Rank(index uint) uint { func (b *BitSet) Select(index uint) uint { leftover := index for idx, word := range b.set { - w := uint(popcount(word)) + w := uint(bits.OnesCount64(word)) if w > leftover { return uint(idx)*64 + select64(word, leftover) } @@ -1182,3 +1506,322 @@ func (b *BitSet) Select(index uint) uint { } return b.length } + +// top detects the top bit set +func (b *BitSet) top() (uint, bool) { + for idx := len(b.set) - 1; idx >= 0; idx-- { + if word := b.set[idx]; word != 0 { + return uint(idx<= b.length { + b.length = top + bits + 1 + } + + pad, idx := top%wordSize, top>>log2WordSize + shift, pages := bits%wordSize, bits>>log2WordSize + if bits%wordSize == 0 { // happy case: just add pages + copy(dst[pages:nsize], b.set) + } else { + if pad+shift >= wordSize { + dst[idx+pages+1] = b.set[idx] >> (wordSize - shift) + } + + for i := int(idx); i >= 0; i-- { + if i > 0 { + dst[i+int(pages)] = (b.set[i] << shift) | (b.set[i-1] >> (wordSize - shift)) + } else { + dst[i+int(pages)] = b.set[i] << shift + } + } + } + + // zeroing extra pages + for i := 0; i < int(pages); i++ { + dst[i] = 0 + } + + b.set = dst +} + +// ShiftRight shifts the bitset like >> operation would do. +func (b *BitSet) ShiftRight(bits uint) { + panicIfNull(b) + + if bits == 0 { + return + } + + top, ok := b.top() + if !ok { + return + } + + if bits > top { + b.set = make([]uint64, wordsNeeded(b.length)) + return + } + + pad, idx := top%wordSize, top>>log2WordSize + shift, pages := bits%wordSize, bits>>log2WordSize + if bits%wordSize == 0 { // happy case: just clear pages + b.set = b.set[pages:] + b.length -= pages * wordSize + } else { + for i := 0; i <= int(idx-pages); i++ { + if i < int(idx-pages) { + b.set[i] = (b.set[i+int(pages)] >> shift) | (b.set[i+int(pages)+1] << (wordSize - shift)) + } else { + b.set[i] = b.set[i+int(pages)] >> shift + } + } + + if pad < shift { + b.set[int(idx-pages)] = 0 + } + } + + for i := int(idx-pages) + 1; i < len(b.set); i++ { + b.set[i] = 0 + } +} + +// OnesBetween returns the number of set bits in the range [from, to). +// The range is inclusive of 'from' and exclusive of 'to'. +// Returns 0 if from >= to. +func (b *BitSet) OnesBetween(from, to uint) uint { + panicIfNull(b) + + if from >= to { + return 0 + } + + // Calculate indices and masks for the starting and ending words + startWord := from >> log2WordSize // Divide by wordSize + endWord := to >> log2WordSize + startOffset := from & wordMask // Mod wordSize + endOffset := to & wordMask + + // Case 1: Bits lie within a single word + if startWord == endWord { + // Create mask for bits between from and to + mask := uint64((1<= startOffset + count = uint(bits.OnesCount64(b.set[startWord] & startMask)) + + // 2b: Count all bits in complete words between start and end + if endWord > startWord+1 { + count += uint(popcntSlice(b.set[startWord+1 : endWord])) + } + + // 2c: Count bits in last word (from start of word to endOffset) + if endOffset > 0 { + endMask := uint64(1<> log2WordSize + bitOffset := outPos & wordMask + + // Write extracted bits, handling word boundary crossing + dst.set[wordIdx] |= extracted << bitOffset + if bitOffset+bitsExtracted > wordSize { + dst.set[wordIdx+1] = extracted >> (wordSize - bitOffset) + } + + outPos += bitsExtracted + } +} + +// Deposit creates a new BitSet and deposits bits according to a mask. +// See DepositTo for details. +func (b *BitSet) Deposit(mask *BitSet) *BitSet { + dst := New(mask.length) + b.DepositTo(mask, dst) + return dst +} + +// DepositTo spreads bits from a compacted form in the BitSet into positions +// specified by mask in dst. This is the inverse operation of Extract. +// +// For example, if mask has bits set at positions 1,4,5, then DepositTo will +// take consecutive bits 0,1,2 from the source BitSet and place them into +// positions 1,4,5 in the destination BitSet. +func (b *BitSet) DepositTo(mask *BitSet, dst *BitSet) { + panicIfNull(b) + panicIfNull(mask) + panicIfNull(dst) + + if len(dst.set) == 0 || len(mask.set) == 0 || len(b.set) == 0 { + return + } + + inPos := uint(0) + length := len(mask.set) + if len(dst.set) < length { + length = len(dst.set) + } + + // Process each word + for i := 0; i < length; i++ { + if mask.set[i] == 0 { + continue // Skip words with no bits to deposit + } + + // Calculate source word index + wordIdx := inPos >> log2WordSize + if wordIdx >= uint(len(b.set)) { + break // No more source bits available + } + + // Get source bits, handling word boundary crossing + sourceBits := b.set[wordIdx] + bitOffset := inPos & wordMask + if wordIdx+1 < uint(len(b.set)) && bitOffset != 0 { + // Combine bits from current and next word + sourceBits = (sourceBits >> bitOffset) | + (b.set[wordIdx+1] << (wordSize - bitOffset)) + } else { + sourceBits >>= bitOffset + } + + // Deposit bits according to mask + dst.set[i] = (dst.set[i] &^ mask.set[i]) | pdep(sourceBits, mask.set[i]) + inPos += uint(bits.OnesCount64(mask.set[i])) + } +} + +//go:generate go run cmd/pextgen/main.go -pkg=bitset + +func pext(w, m uint64) (result uint64) { + var outPos uint + + // Process byte by byte + for i := 0; i < 8; i++ { + shift := i << 3 // i * 8 using bit shift + b := uint8(w >> shift) + mask := uint8(m >> shift) + + extracted := pextLUT[b][mask] + bits := popLUT[mask] + + result |= uint64(extracted) << outPos + outPos += uint(bits) + } + + return result +} + +func pdep(w, m uint64) (result uint64) { + var inPos uint + + // Process byte by byte + for i := 0; i < 8; i++ { + shift := i << 3 // i * 8 using bit shift + mask := uint8(m >> shift) + bits := popLUT[mask] + + // Get the bits we'll deposit from the source + b := uint8(w >> inPos) + + // Deposit them according to the mask for this byte + deposited := pdepLUT[b][mask] + + // Add to result + result |= uint64(deposited) << shift + inPos += uint(bits) + } + + return result +} diff --git a/vendor/github.com/bits-and-blooms/bitset/bitset_iter.go b/vendor/github.com/bits-and-blooms/bitset/bitset_iter.go new file mode 100644 index 0000000000..2765fe741d --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/bitset_iter.go @@ -0,0 +1,31 @@ +//go:build go1.23 +// +build go1.23 + +package bitset + +import ( + "iter" + "math/bits" +) + +// EachSet returns an iterator over the indices of all set bits in ascending +// order, for use with Go 1.23+ range-over-function loops: +// +// for i := range b.EachSet() { +// // i is the index of a set bit +// } +// +// Breaking out of the loop stops the iteration early. +func (b *BitSet) EachSet() iter.Seq[uint] { + return func(yield func(uint) bool) { + for wordIndex, word := range b.set { + idx := 0 + for trail := bits.TrailingZeros64(word); trail != 64; trail = bits.TrailingZeros64(word >> idx) { + if !yield(uint(wordIndex<> 1) & 0x5555555555555555 - x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 - x += x >> 4 - x &= 0x0f0f0f0f0f0f0f0f - x *= 0x0101010101010101 - return x >> 56 -} +import "math/bits" -func popcntSliceGo(s []uint64) uint64 { - cnt := uint64(0) - for _, x := range s { - cnt += popcount(x) +func popcntSlice(s []uint64) (cnt uint64) { + // Unroll the loop with four independent accumulators to break the + // dependency chain on cnt and let the CPU run multiple popcounts in + // parallel. + var c0, c1, c2, c3 uint64 + n := len(s) + i := 0 + for ; i <= n-4; i += 4 { + c0 += uint64(bits.OnesCount64(s[i])) + c1 += uint64(bits.OnesCount64(s[i+1])) + c2 += uint64(bits.OnesCount64(s[i+2])) + c3 += uint64(bits.OnesCount64(s[i+3])) + } + cnt = c0 + c1 + c2 + c3 + for ; i < n; i++ { + cnt += uint64(bits.OnesCount64(s[i])) } - return cnt + return } -func popcntMaskSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcount(s[i] &^ m[i]) +func popcntMaskSlice(s, m []uint64) (cnt uint64) { + // The next line is to help the bounds checker, it matters! + _ = m[len(s)-1] // BCE + // Four independent accumulators break the dependency chain on cnt + // so the CPU can run multiple popcounts in parallel. + var c0, c1, c2, c3 uint64 + n := len(s) + i := 0 + for ; i <= n-4; i += 4 { + c0 += uint64(bits.OnesCount64(s[i] &^ m[i])) + c1 += uint64(bits.OnesCount64(s[i+1] &^ m[i+1])) + c2 += uint64(bits.OnesCount64(s[i+2] &^ m[i+2])) + c3 += uint64(bits.OnesCount64(s[i+3] &^ m[i+3])) + } + cnt = c0 + c1 + c2 + c3 + for ; i < n; i++ { + cnt += uint64(bits.OnesCount64(s[i] &^ m[i])) } - return cnt + return } -func popcntAndSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) +// popcntAndSlice computes the population count of the AND of two slices. +// It assumes that len(m) >= len(s) > 0. +func popcntAndSlice(s, m []uint64) (cnt uint64) { + // The next line is to help the bounds checker, it matters! + _ = m[len(s)-1] // BCE for i := range s { - cnt += popcount(s[i] & m[i]) + cnt += uint64(bits.OnesCount64(s[i] & m[i])) } - return cnt + return } -func popcntOrSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) +// popcntOrSlice computes the population count of the OR of two slices. +// It assumes that len(m) >= len(s) > 0. +func popcntOrSlice(s, m []uint64) (cnt uint64) { + // The next line is to help the bounds checker, it matters! + _ = m[len(s)-1] // BCE for i := range s { - cnt += popcount(s[i] | m[i]) + cnt += uint64(bits.OnesCount64(s[i] | m[i])) } - return cnt + return } -func popcntXorSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) +// popcntXorSlice computes the population count of the XOR of two slices. +// It assumes that len(m) >= len(s) > 0. +func popcntXorSlice(s, m []uint64) (cnt uint64) { + // The next line is to help the bounds checker, it matters! + _ = m[len(s)-1] // BCE for i := range s { - cnt += popcount(s[i] ^ m[i]) + cnt += uint64(bits.OnesCount64(s[i] ^ m[i])) } - return cnt + return } diff --git a/vendor/github.com/bits-and-blooms/bitset/popcnt_19.go b/vendor/github.com/bits-and-blooms/bitset/popcnt_19.go deleted file mode 100644 index 7855c04b5b..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/popcnt_19.go +++ /dev/null @@ -1,62 +0,0 @@ -//go:build go1.9 -// +build go1.9 - -package bitset - -import "math/bits" - -func popcntSlice(s []uint64) uint64 { - var cnt int - for _, x := range s { - cnt += bits.OnesCount64(x) - } - return uint64(cnt) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - var cnt int - // this explicit check eliminates a bounds check in the loop - if len(m) < len(s) { - panic("mask slice is too short") - } - for i := range s { - cnt += bits.OnesCount64(s[i] &^ m[i]) - } - return uint64(cnt) -} - -func popcntAndSlice(s, m []uint64) uint64 { - var cnt int - // this explicit check eliminates a bounds check in the loop - if len(m) < len(s) { - panic("mask slice is too short") - } - for i := range s { - cnt += bits.OnesCount64(s[i] & m[i]) - } - return uint64(cnt) -} - -func popcntOrSlice(s, m []uint64) uint64 { - var cnt int - // this explicit check eliminates a bounds check in the loop - if len(m) < len(s) { - panic("mask slice is too short") - } - for i := range s { - cnt += bits.OnesCount64(s[i] | m[i]) - } - return uint64(cnt) -} - -func popcntXorSlice(s, m []uint64) uint64 { - var cnt int - // this explicit check eliminates a bounds check in the loop - if len(m) < len(s) { - panic("mask slice is too short") - } - for i := range s { - cnt += bits.OnesCount64(s[i] ^ m[i]) - } - return uint64(cnt) -} diff --git a/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.go b/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.go deleted file mode 100644 index 116e044407..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.go +++ /dev/null @@ -1,68 +0,0 @@ -//go:build !go1.9 && amd64 && !appengine -// +build !go1.9,amd64,!appengine - -package bitset - -// *** the following functions are defined in popcnt_amd64.s - -//go:noescape - -func hasAsm() bool - -// useAsm is a flag used to select the GO or ASM implementation of the popcnt function -var useAsm = hasAsm() - -//go:noescape - -func popcntSliceAsm(s []uint64) uint64 - -//go:noescape - -func popcntMaskSliceAsm(s, m []uint64) uint64 - -//go:noescape - -func popcntAndSliceAsm(s, m []uint64) uint64 - -//go:noescape - -func popcntOrSliceAsm(s, m []uint64) uint64 - -//go:noescape - -func popcntXorSliceAsm(s, m []uint64) uint64 - -func popcntSlice(s []uint64) uint64 { - if useAsm { - return popcntSliceAsm(s) - } - return popcntSliceGo(s) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - if useAsm { - return popcntMaskSliceAsm(s, m) - } - return popcntMaskSliceGo(s, m) -} - -func popcntAndSlice(s, m []uint64) uint64 { - if useAsm { - return popcntAndSliceAsm(s, m) - } - return popcntAndSliceGo(s, m) -} - -func popcntOrSlice(s, m []uint64) uint64 { - if useAsm { - return popcntOrSliceAsm(s, m) - } - return popcntOrSliceGo(s, m) -} - -func popcntXorSlice(s, m []uint64) uint64 { - if useAsm { - return popcntXorSliceAsm(s, m) - } - return popcntXorSliceGo(s, m) -} diff --git a/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.s b/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.s deleted file mode 100644 index 666c0dcc17..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.s +++ /dev/null @@ -1,104 +0,0 @@ -// +build !go1.9 -// +build amd64,!appengine - -TEXT ·hasAsm(SB),4,$0-1 -MOVQ $1, AX -CPUID -SHRQ $23, CX -ANDQ $1, CX -MOVB CX, ret+0(FP) -RET - -#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 - -TEXT ·popcntSliceAsm(SB),4,$0-32 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntSliceEnd -popcntSliceLoop: -BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX -ADDQ DX, AX -ADDQ $8, SI -LOOP popcntSliceLoop -popcntSliceEnd: -MOVQ AX, ret+24(FP) -RET - -TEXT ·popcntMaskSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntMaskSliceEnd -MOVQ m+24(FP), DI -popcntMaskSliceLoop: -MOVQ (DI), DX -NOTQ DX -ANDQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntMaskSliceLoop -popcntMaskSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntAndSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntAndSliceEnd -MOVQ m+24(FP), DI -popcntAndSliceLoop: -MOVQ (DI), DX -ANDQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntAndSliceLoop -popcntAndSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntOrSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntOrSliceEnd -MOVQ m+24(FP), DI -popcntOrSliceLoop: -MOVQ (DI), DX -ORQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntOrSliceLoop -popcntOrSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntXorSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntXorSliceEnd -MOVQ m+24(FP), DI -popcntXorSliceLoop: -MOVQ (DI), DX -XORQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntXorSliceLoop -popcntXorSliceEnd: -MOVQ AX, ret+48(FP) -RET diff --git a/vendor/github.com/bits-and-blooms/bitset/popcnt_generic.go b/vendor/github.com/bits-and-blooms/bitset/popcnt_generic.go deleted file mode 100644 index 9e0ad464e0..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/popcnt_generic.go +++ /dev/null @@ -1,25 +0,0 @@ -//go:build !go1.9 && (!amd64 || appengine) -// +build !go1.9 -// +build !amd64 appengine - -package bitset - -func popcntSlice(s []uint64) uint64 { - return popcntSliceGo(s) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - return popcntMaskSliceGo(s, m) -} - -func popcntAndSlice(s, m []uint64) uint64 { - return popcntAndSliceGo(s, m) -} - -func popcntOrSlice(s, m []uint64) uint64 { - return popcntOrSliceGo(s, m) -} - -func popcntXorSlice(s, m []uint64) uint64 { - return popcntXorSliceGo(s, m) -} diff --git a/vendor/github.com/bits-and-blooms/bitset/run_fuzz_tests.sh b/vendor/github.com/bits-and-blooms/bitset/run_fuzz_tests.sh new file mode 100644 index 0000000000..a1d3c72b05 --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/run_fuzz_tests.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Script to run all bitset fuzz tests for a configurable duration or number of iterations. +# Usage: ./run_fuzz_tests.sh [duration/iterations] +# Examples: +# ./run_fuzz_tests.sh # Uses default 60s +# ./run_fuzz_tests.sh 10s # Run each test for 10 seconds +# ./run_fuzz_tests.sh 1000x # Run each test for 1000 iterations + +# Set default duration/iterations if not provided +FUZZTIME=${1:-60s} + +echo "Running BitSet fuzz tests for: $FUZZTIME" +echo "" + +# List of all available fuzz tests +FUZZ_TESTS=( + FuzzCapacityAndGrowth + FuzzIterationConsistency + FuzzStringRepresentations + FuzzBasicOps + FuzzRange + FuzzSetOperations + FuzzNavigation + FuzzShift + FuzzModification + FuzzCopy + FuzzSerialization + FuzzRandomOperations +) + +# Run each fuzz test +for test in "${FUZZ_TESTS[@]}"; do + echo "Testing $test..." + go test -fuzz=$test -fuzztime=$FUZZTIME + echo "" +done + +echo "" +echo "All available fuzz tests completed for: $FUZZTIME" diff --git a/vendor/github.com/bits-and-blooms/bitset/select.go b/vendor/github.com/bits-and-blooms/bitset/select.go index f15e74a2c9..a43c6bd6a2 100644 --- a/vendor/github.com/bits-and-blooms/bitset/select.go +++ b/vendor/github.com/bits-and-blooms/bitset/select.go @@ -1,10 +1,12 @@ package bitset +import "math/bits" + func select64(w uint64, j uint) uint { seen := 0 // Divide 64bit part := w & 0xFFFFFFFF - n := uint(popcount(part)) + n := uint(bits.OnesCount64(part)) if n <= j { part = w >> 32 seen += 32 @@ -15,7 +17,7 @@ func select64(w uint64, j uint) uint { // Divide 32bit part = ww & 0xFFFF - n = uint(popcount(part)) + n = uint(bits.OnesCount64(part)) if n <= j { part = ww >> 16 seen += 16 @@ -25,7 +27,7 @@ func select64(w uint64, j uint) uint { // Divide 16bit part = ww & 0xFF - n = uint(popcount(part)) + n = uint(bits.OnesCount64(part)) if n <= j { part = ww >> 8 seen += 8 diff --git a/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_18.go b/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_18.go deleted file mode 100644 index 12336e76af..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_18.go +++ /dev/null @@ -1,15 +0,0 @@ -//go:build !go1.9 -// +build !go1.9 - -package bitset - -var deBruijn = [...]byte{ - 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, - 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, - 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, - 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, -} - -func trailingZeroes64(v uint64) uint { - return uint(deBruijn[((v&-v)*0x03f79d71b4ca8b09)>>58]) -} diff --git a/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_19.go b/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_19.go deleted file mode 100644 index cfb0a84091..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_19.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build go1.9 -// +build go1.9 - -package bitset - -import "math/bits" - -func trailingZeroes64(v uint64) uint { - return uint(bits.TrailingZeros64(v)) -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 22a5f74188..d014545d5c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -12,7 +12,7 @@ github.com/Microsoft/go-winio/pkg/guid # github.com/beorn7/perks v1.0.1 ## explicit; go 1.11 github.com/beorn7/perks/quantile -# github.com/bits-and-blooms/bitset v1.13.0 +# github.com/bits-and-blooms/bitset v1.25.0 ## explicit; go 1.16 github.com/bits-and-blooms/bitset # github.com/cespare/xxhash/v2 v2.3.0