diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 230564c..6a472c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,10 @@ jobs: repo-token: ${{ github.token }} - name: Test run: cargo test --locked --all-features + - name: Build API documentation + env: + RUSTDOCFLAGS: "-D warnings" + run: cargo doc --locked --all-features --no-deps - name: Check all targets run: cargo check --locked --all-targets --all-features - name: Format diff --git a/AGENTS.md b/AGENTS.md index 12b0f72..b15520b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,10 +10,12 @@ Unirust is a distributed temporal entity resolution engine. The primary function ### Entity Resolution Must Always Happen -Every ingested record MUST go through entity resolution. Never skip or bypass: -- `linker.link_records_batch_parallel()` - batch links with parallel extraction -- `partitioned.process_batch_optimized()` - optimized partition processing -- `partitioned.ingest_batch()` - distributed batch processing +Every newly accepted record MUST complete entity resolution before its record data +commits. Valid retries of an already resolved source identity remain idempotent. +Preserve the linker calls in every ingest path, including: +- `Unirust::stream_records()` — persistent shard ingestion +- `StreamingLinker::link_records_batch_parallel_with_interner()` — parallel extraction and sequential linking +- `Partition::process_batch_optimized()` — the separate in-memory partition path Any optimization that skips entity resolution is incorrect and breaks the core value proposition. @@ -53,17 +55,23 @@ src/ ## Key Entry Points -### Ingest Flow -1. `distributed.rs:ShardNode::ingest_records()` - gRPC entry -2. `distributed.rs:dispatch_ingest_partitioned()` - routes to partitioned processing -3. `partitioned.rs:ParallelPartitionedUnirust::ingest_batch_with_partitions()` - parallel partition dispatch -4. `partitioned.rs:Partition::process_batch_optimized()` - **hot path**: batch insert → parallel extract → sequential link -5. `linker.rs:link_records_batch_parallel()` - parallel key extraction, sequential DSU merges +### Persistent Ingest Flow +1. `distributed.rs:RouterService::ingest_records()` — placement and source-identity reservations +2. `distributed.rs:ShardNode::ingest_records()` — shard RPC and ingest WAL +3. `distributed.rs:dispatch_ingest_records()` — worker dispatch +4. `lib.rs:Unirust::stream_records()` — stage records, link, commit and flush +5. `linker.rs:StreamingLinker` — key extraction, temporal guards and DSU merges + +Persistent shards disable partitioned ingestion. `dispatch_ingest_partitioned()` +and `ParallelPartitionedUnirust` use in-memory partition stores and must never +replace the persistent path. ### Query Flow -1. `distributed.rs:RouterService::query_entities()` - gRPC entry -2. `lib.rs:Unirust::query_master_entities()` - query execution -3. `query.rs` - query planning and execution +1. `distributed.rs:RouterService::query_entities()` — concurrent candidate discovery and canonical fragment hydration +2. `lib.rs:Unirust::query_master_entities()` — local candidates and golden descriptors +3. `query.rs` and `graph.rs` — temporal conjunction and mastering + +See [DESIGN.md](DESIGN.md) for recovery, reconciliation and matching limits. ## Testing Strategy @@ -78,19 +86,31 @@ src/ - Test distributed scenarios (router + shards) ### Load Testing -- Use `unirust_loadtest` binary -- Standard command: `./target/release/unirust_loadtest -r http://127.0.0.1:50060 -c 10000000 --streams 16 --batch 5000` -- Baseline with 5 shards, 10% overlap: **~410K rec/sec, ~12ms batch latency** + +Build the load generator explicitly with `--features test-support` (see below). +Use fresh persistent directories and a fixed ontology, topology, seed and workload +for comparisons. Do not change the shard count of an existing dataset to run a +benchmark; durable reservations bind it to its topology. ## Performance Considerations ### Do Not Regress -After any change, verify performance with loadtest. Current baseline with 5 shards: -- **~410K records/second** (10% overlap) -- **~12ms batch latency** + +For runtime changes, compare persistent load tests before and after the change, +including acknowledgement counts, failures, throughput and measured RPC latency. +Use focused diagnostics for query or recovery changes. Documentation-only edits +require checking their examples and claims rather than repeating throughput runs. + +The September audit measured approximately 43,415 versus 43,939 records/second +on one local five-shard, one-million-record comparison (10% overlap); average RPC +latency was approximately 1,585 versus 1,657 ms. These are observations, not +performance guarantees. The old 410K records/second and 12 ms figures do not +establish a durable production baseline. Workload details and limitations are in +[the audit report](docs/critical-audit-2026-09-05.md). ### Hot Paths -- `partitioned.rs:process_batch_optimized()` - batch insert + parallel extract + sequential link +- `lib.rs:stream_records()` - persistent staging, resolution and commit +- `partitioned.rs:process_batch_optimized()` - separate in-memory partition processing - `linker.rs:link_records_batch_parallel()` - parallel extraction, sequential DSU - `linker.rs:link_extracted_record()` - DSU merges with temporal guards - `dsu.rs:find()` - path compression with root cache @@ -126,15 +146,17 @@ After any change, verify performance with loadtest. Current baseline with 5 shar ```bash # Development -cargo test # Run all tests -cargo clippy --all-targets # Lint -cargo fmt # Format +cargo test --locked --all-features +cargo clippy --locked --all-targets --all-features -- -D warnings +cargo fmt --check +cargo test --doc --locked --all-features # Benchmarks -cargo bench --bench bench_quick # Fast (~30s) +cargo bench --bench bench_quick # Focused benchmark suite cargo bench --bench bench_micro # Component benchmarks -# Start cluster (recommended) +# Build the load generator; run the cluster on a fresh benchmark dataset +cargo build --release --locked --features test-support --bin unirust_loadtest SHARDS=5 ./scripts/cluster.sh start # Load test (requires running cluster) @@ -142,7 +164,7 @@ SHARDS=5 ./scripts/cluster.sh start --router http://127.0.0.1:50060 \ --count 10000000 \ --streams 16 \ - --batch 5000 + --batch 5000 --overlap 0.1 --seed 42 # Stop cluster ./scripts/cluster.sh stop @@ -150,7 +172,7 @@ SHARDS=5 ./scripts/cluster.sh start ## Style Guidelines -- Use `Result` for fallible operations +- Follow existing error types: `anyhow::Result` for library operations and `tonic::Status` at gRPC boundaries - Prefer `&str` over `String` for parameters - Use `#[inline]` for small hot functions - Avoid `unwrap()` in library code diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e713e7..8e39c38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,24 @@ All notable changes to this project are documented here. This project follows Semantic Versioning. +## [Unreleased] + +### Fixed + +- Authenticate protoc setup requests in CI to avoid unauthenticated GitHub API + rate limits (merged after the v0.2.0 tag). + +### Documentation + +- Align deployment, architecture, configuration and Rust API examples with the + persistent runtime; describe matching, memory and recovery limits explicitly. +- Correct load-test diagnostic guidance: `UNIRUST_PROFILE` selects a tuning + preset and does not enable a profiler. +- Build API documentation in CI with warnings treated as errors; compile the + crate quick-start example as a doctest. +- Remove unsupported GPU projections and performance guarantees, and distinguish + measured local diagnostics from production capacity claims. + ## [0.2.0] - 2026-09-05 ### Added @@ -92,5 +110,7 @@ Semantic Versioning. - Distributed integration tests now use temporary persistent shard stores; the in-memory store remains limited to unit tests. - Replaced the non-durable historical performance claim with a verified - five-shard, power-loss-durable baseline of 50,598 records/second for the - documented 10-million-record workload. + five-shard measurement reported at release preparation: 50,598 records/second + for a 10-million-record workload using synchronous durable ingestion. This + historical result is not a capacity guarantee or a physical power-cut test; + the later comparative audit used a separate one-million-record workload. diff --git a/Containerfile b/Containerfile index 17944fc..dbb41e2 100644 --- a/Containerfile +++ b/Containerfile @@ -1,13 +1,15 @@ # Unirust Container Image # -# Multi-stage build for efficient, minimal production image +# Multi-stage Rust build with a Debian slim runtime and CLI tools # # Usage: # podman build -t unirust -f Containerfile . -# podman run --rm -p 50061:50061 -v unirust-data:/data -v unirust-backup:/backup unirust shard -# podman run --rm -p 50060:50060 unirust router --shards shard-0:50061 +# podman run --rm -p 127.0.0.1:50061:50061 -v unirust-data:/data -v unirust-backup:/backup unirust +# The default command includes --data-dir /data and --backup-dir /backup. +# Supplying an explicit command replaces those defaults; pass both paths to shard. # -# Or use with compose.yaml for full cluster deployment +# Use compose.yaml or scripts/podman_cluster.sh for a router/shard network. +# Separate named data/backup volumes can still share one host or physical disk. FROM rust:1.88-bookworm AS builder diff --git a/DESIGN.md b/DESIGN.md index 66b6c78..2b957bb 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,733 +1,357 @@ -# Unirust Architecture & Design - -This document describes the internal architecture, algorithms, and design decisions of Unirust. It serves as the authoritative reference for understanding how the system works. - -## Table of Contents - -1. [System Overview](#system-overview) -2. [Core Concepts](#core-concepts) -3. [Entity Resolution Algorithm](#entity-resolution-algorithm) -4. [Conflict Detection](#conflict-detection) -5. [Distributed Architecture](#distributed-architecture) -6. [Cross-Shard Reconciliation](#cross-shard-reconciliation) -7. [Storage Layer](#storage-layer) -8. [Performance Optimizations](#performance-optimizations) -9. [Data Flow](#data-flow) - ---- - -## System Overview - -Unirust is a temporal entity resolution engine that clusters records from multiple source systems into unified master entities. The system supports both single-node and distributed deployments. - -``` - ┌─────────────────┐ - │ Clients │ - │ (gRPC/API) │ - └────────┬────────┘ - │ - ┌────────▼────────┐ - │ Router │ - │ (hash-based │ - │ routing) │ - └────────┬────────┘ - │ - ┌────────────────────┼────────────────────┐ - │ │ │ - ┌───────▼───────┐ ┌───────▼───────┐ ┌───────▼───────┐ - │ Shard 0 │ │ Shard 1 │ │ Shard N │ - │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ - │ │ Linker │ │ │ │ Linker │ │ │ │ Linker │ │ - │ │ DSU │ │ │ │ DSU │ │ │ │ DSU │ │ - │ │ Index │ │ │ │ Index │ │ │ │ Index │ │ - │ │ RocksDB │ │ │ │ RocksDB │ │ │ │ RocksDB │ │ - │ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ - └───────────────┘ └───────────────┘ └───────────────┘ -``` - ---- - -## Core Concepts - -### Records - -A **Record** represents a single observation from a source system: - -```rust -struct Record { - id: RecordId, // Unique within shard - identity: RecordIdentity, // Entity type + perspective + UID - descriptors: Vec, // Attribute-value pairs with time intervals -} - -struct RecordIdentity { - entity_type: String, // e.g., "person", "company" - perspective: String, // Source system, e.g., "crm", "erp" - uid: String, // Source-unique identifier -} - -struct Descriptor { - attr: AttrId, // Interned attribute name - value: ValueId, // Interned value - interval: Interval, // [start, end) validity period -} -``` - -### Ontology - -The **Ontology** defines matching rules: - -1. **Identity Keys**: Attribute combinations that identify the same entity - ```rust - // Records with matching (name, email) are considered the same entity - IdentityKey::new(vec![name_attr, email_attr], "name_email") - ``` - -2. **Strong Identifiers**: Attributes that cannot conflict within a cluster - ```rust - // SSN must be unique - conflicting SSNs block merges - StrongIdentifier::new(ssn_attr, "ssn_unique") - ``` - -3. **Constraints**: Validation rules - ```rust - Constraint::unique(email_attr, "unique_email") - Constraint::unique_within_perspective(account_id, "source_unique") - ``` - -### Temporal Model - -All data has temporal validity. An **Interval** `[start, end)` defines when a descriptor is valid. Entity resolution respects these intervals: - -- Records only merge if their identity key values match during **overlapping time periods** -- Conflicts are detected per-interval, not globally -- Golden records are computed for each unique time period - ---- - -## Entity Resolution Algorithm - -### Streaming Linker - -The core algorithm uses **batch-parallel processing** for high throughput. Within each partition, records are processed in three phases: - -``` -Phase 1: Batch Store Insertion - │ All records added to store, collecting (index, record_id) pairs - ▼ -Phase 2: Parallel Extraction (Rayon) - │ Extract identity keys, strong ID summaries across all records - ▼ -Phase 3: Sequential Linking - │ DSU merges with temporal guards (serialized for correctness) - ▼ -Result: Cluster assignments returned in original batch order -``` - -Parallel extraction dominates compute time; DSU mutations are inherently sequential due to data dependencies but benefit from root caching. - -#### Phase 1: Key Extraction - -For each record, extract values for all identity keys defined in the ontology: - -```rust -fn extract_key_values(record: &Record, ontology: &Ontology) -> Vec { - ontology.identity_keys() - .iter() - .filter_map(|key| { - // Collect all attribute values for this identity key - let values: Vec<_> = key.attributes() - .iter() - .filter_map(|attr| record.value_for(*attr)) - .collect(); - - // Only complete keys (all attributes present) form valid key values - if values.len() == key.attributes().len() { - Some(KeyValue::new(key.name(), values)) - } else { - None - } - }) - .collect() -} -``` - -#### Phase 2: Candidate Discovery - -For each key value, query the identity index: - -```rust -fn find_candidates(key_value: &KeyValue, index: &IdentityIndex) -> Vec { - let signature = hash_key_value(key_value); - index.get(&signature).unwrap_or_default() -} -``` - -The identity index maps key value signatures to record IDs. This is the hot path, optimized with: -- Bloom filters for fast negative lookups (16MB filter, <1% false positive rate) -- Sharded caching to avoid lock contention -- SIMD-accelerated hashing - -#### Phase 3: Cluster Merging - -The Disjoint Set Union (DSU) data structure tracks cluster membership. Merging follows these rules: - -```rust -fn try_merge(dsu: &mut TemporalDSU, a: RecordId, b: RecordId, - store: &Store, ontology: &Ontology) -> MergeResult { - let root_a = dsu.find(a); - let root_b = dsu.find(b); - - if root_a == root_b { - return MergeResult::AlreadySame; - } - - // Check temporal guards - let guard = compute_temporal_guard(root_a, root_b, store, ontology); - - match guard { - TemporalGuard::Allowed { reason, interval } => { - dsu.union_with_guard(root_a, root_b, guard); - MergeResult::Merged { interval } - } - TemporalGuard::Blocked { reason } => { - MergeResult::Conflict { reason } - } - } -} -``` - -**Temporal Guards** validate merges: - -1. **Overlapping Intervals**: Records must have overlapping validity periods -2. **Strong Identifier Agreement**: Strong identifiers must match during overlap -3. **Constraint Satisfaction**: Uniqueness constraints must be satisfied - -#### Phase 4: Assignment Finalization - -Each record receives its final cluster ID: - -```rust -fn finalize_assignment(record_id: RecordId, dsu: &TemporalDSU) -> ClusterId { - ClusterId(dsu.find(record_id).0) -} -``` - -### Adaptive Candidate Capping - -To prevent pathological cases (e.g., very common names), candidate discovery uses adaptive capping: - -| Candidate Count | Cap Applied | -|-----------------|-------------| -| < 2,000 | No cap | -| 2,000 - 10,000 | Cap at 1,000 | -| > 10,000 | Cap at 500 | -| > 50,000 | Early exit (hot key) | - -Additionally, **stochastic sampling** maintains match quality: when candidates exceed the threshold, random sampling weighted by temporal overlap preserves expected accuracy. - ---- - -## Conflict Detection - -Conflicts occur when records in the same cluster have incompatible values for the same attribute during overlapping time periods. - -### Detection Algorithms - -Unirust implements two conflict detection algorithms with automatic selection: - -#### 1. Sweep-Line Algorithm (O(n log n)) - -Best for clusters with diverse time boundaries. - -``` -Events: [(t1, START, r1), (t2, END, r1), (t3, START, r2), ...] - sorted by time - -Active set: records currently "open" - -For each event: - if START: add to active set, check conflicts with all active - if END: remove from active set -``` - -#### 2. Atomic Intervals Algorithm (O(atoms × n)) - -Best for clusters with high overlap (many records share same intervals). - -``` -1. Collect all unique time boundaries: {t1, t2, t3, ...} -2. Create atomic intervals: [t1,t2), [t2,t3), [t3,t4), ... -3. For each atomic interval: - - Find all records active during this interval - - Group by attribute - - If multiple values for same attribute → conflict -``` - -#### Auto-Selection Heuristic - -```rust -fn select_algorithm(unique_boundaries: usize, total_descriptors: usize) -> Algorithm { - let max_boundaries = total_descriptors * 2; // Each descriptor has start + end - let ratio = unique_boundaries as f64 / max_boundaries as f64; - - if ratio < 0.5 { - // High overlap → atomic intervals is faster - Algorithm::AtomicIntervals - } else { - // Low overlap → sweep line is faster - Algorithm::SweepLine - } -} -``` - -### Conflict Types - -1. **Direct Conflict**: Same attribute has different values in overlapping intervals - ``` - Record A: email = "john@foo.com" [100, 200) - Record B: email = "john@bar.com" [150, 250) - → Conflict in [150, 200) - ``` - -2. **Indirect Conflict**: Strong identifier violation - ``` - Record A: ssn = "123-45-6789" [100, 200) - Record B: ssn = "987-65-4321" [150, 250) - → Blocked merge (strong identifier conflict) - ``` - ---- - -## Distributed Architecture - -### Router - -The router provides the external API and routes requests to shards: - -```rust -impl Router { - async fn ingest(&self, records: Vec) -> Vec { - // Group records by target shard - let mut shard_batches: HashMap> = HashMap::new(); - - for record in records { - let shard_id = self.route(&record); - shard_batches.entry(shard_id).or_default().push(record); - } - - // Fan out to shards in parallel - let futures: Vec<_> = shard_batches - .into_iter() - .map(|(shard_id, batch)| { - self.shards[shard_id].ingest(batch) - }) - .collect(); - - // Collect results - join_all(futures).await.into_iter().flatten().collect() - } -} -``` - -### Routing Strategy - -Records are routed by hashing their identity key values: - -```rust -fn route(record: &Record, num_shards: usize) -> ShardId { - let key_values = extract_key_values(record); - let hash = hash_key_values(&key_values); - ShardId(hash % num_shards as u64) -} -``` - -This ensures records that might need to merge are routed to the same shard. - -### Global Cluster IDs - -Each cluster has a globally unique ID: - -``` -┌──────────────────────────────────────────────────────────────┐ -│ GlobalClusterId (64 bits) │ -├──────────────┬────────────────────┬──────────────────────────┤ -│ shard_id │ version │ local_id │ -│ (16 bits) │ (16 bits) │ (32 bits) │ -└──────────────┴────────────────────┴──────────────────────────┘ -``` - -- **shard_id**: Owning shard (0-65535) -- **version**: Merge version for conflict detection -- **local_id**: Cluster ID within the shard - ---- - -## Cross-Shard Reconciliation - -### The Problem - -When records that should merge are routed to different shards, we need cross-shard reconciliation: - -``` -Shard 0 Shard 1 -┌────────────┐ ┌────────────┐ -│ Record A │ │ Record B │ -│ name=John │ Should merge │ name=John │ -│ email=j@x │ ◄───────────────► │ email=j@x │ -│ Cluster 0 │ │ Cluster 5 │ -└────────────┘ └────────────┘ -``` - -### Boundary Tracking - -Each shard maintains a **Cluster Boundary Index** tracking identity keys that appear: - -```rust -struct ClusterBoundaryIndex { - // Maps identity key signature → boundary entries - boundaries: HashMap>, - // Bloom filter for fast negative lookups - bloom: BloomFilter, - // Keys modified since last reconciliation - dirty_keys: HashSet, -} - -struct BoundaryEntry { - cluster_id: GlobalClusterId, - interval: Interval, - shard_id: ShardId, -} -``` - -### Reconciliation Algorithm - -1. **Dirty Key Collection**: Each shard tracks keys modified since last reconciliation - -2. **Boundary Exchange**: Router collects dirty boundaries from all shards - -3. **Merge Detection**: For each key appearing on multiple shards: - ```rust - fn detect_cross_shard_merges(key: &IdentityKeySignature, - entries: &[BoundaryEntry]) -> Vec { - let mut merges = Vec::new(); - - // Group by overlapping intervals - for (a, b) in entries.iter().tuple_combinations() { - if a.shard_id != b.shard_id && a.interval.overlaps(&b.interval) { - merges.push(ClusterMerge { - primary: a.cluster_id, - secondary: b.cluster_id, - }); - } - } - - merges - } - ``` - -4. **Merge Application**: Each shard applies merges to its local DSU: - ```rust - fn apply_cross_shard_merge(&mut self, primary: GlobalClusterId, - secondary: GlobalClusterId) -> usize { - // Update all records in secondary cluster to point to primary - let mut updated = 0; - for record_id in self.records_in_cluster(secondary) { - self.cluster_map.insert(record_id, primary); - updated += 1; - } - updated - } - ``` - -5. **Key Clearing**: Successfully reconciled keys are removed from dirty set - -### Consistency Guarantees - -- **Eventual Consistency**: Cross-shard clusters converge after reconciliation -- **No Data Loss**: Failed reconciliation retries on next cycle -- **Conflict Preservation**: Cross-shard conflicts are detected and reported - ---- - -## Storage Layer - -### In-Memory Store - -For testing and small datasets: - -```rust -struct Store { - records: FxHashMap, - by_entity_type: FxHashMap>, - attr_interner: StringInterner, - value_interner: StringInterner, -} -``` - -### Persistent Store (RocksDB) - -Production storage with column families: - -| Column Family | Key | Value | Purpose | -|---------------|-----|-------|---------| -| `records` | RecordId (4B) | Record (bincode) | Record storage | -| `index_identity` | hash (8B) | RecordId list | Identity key index | -| `index_attr_value` | attr:value | RecordId list | Attribute lookup | -| `dsu_parent` | RecordId | Parent RecordId | DSU parent links | -| `dsu_rank` | RecordId | u32 | DSU rank for balancing | -| `dsu_guards` | (RecordId, RecordId) | TemporalGuard | Merge guards | -| `cluster_assignments` | RecordId | ClusterId | Cluster membership | -| `interner` | String | InternedId | String interning | -| `metadata` | key | value | Manifest, counters | - -### Tuning Parameters - -```toml -[storage] -block_cache_mb = 512 # Read cache -write_buffer_mb = 128 # Write buffer before flush -max_background_jobs = 4 # Compaction threads -rate_limit_mbps = 0 # I/O rate limiting (0 = unlimited) -``` - ---- - -## Performance Optimizations - -### Lock-Free Structures - -1. **Atomic DSU**: Lock-free parent updates using CAS operations - ```rust - fn find(&self, x: RecordId) -> RecordId { - let mut current = x; - loop { - let parent = self.parent[current].load(Ordering::Acquire); - if parent == current { - return current; - } - // Path compression with CAS - let grandparent = self.parent[parent].load(Ordering::Acquire); - let _ = self.parent[current].compare_exchange( - parent, grandparent, Ordering::Release, Ordering::Relaxed - ); - current = parent; - } - } - ``` - -2. **Sharded Caching**: 256 shards to minimize contention - ```rust - fn get_shard(&self, key: &K) -> &RwLock> { - let hash = hash(key); - &self.shards[hash as usize % 256] - } - ``` - -### SIMD Hashing - -Identity key hashing uses SIMD for throughput: - -```rust -fn simd_hash(data: &[u8]) -> u64 { - // Process 32 bytes at a time using AVX2 - let mut state = _mm256_set1_epi64x(SEED); - for chunk in data.chunks_exact(32) { - let block = _mm256_loadu_si256(chunk.as_ptr() as *const _); - state = _mm256_xor_si256(state, block); - state = _mm256_mul_epi32(state, MULTIPLIER); - } - // Horizontal reduction - reduce_256_to_64(state) -} -``` - -### Async WAL - -Write-ahead logging with coalescing: - -```rust -struct AsyncWal { - tx: Sender, // Submit writes - writer_thread: JoinHandle, // Background writer -} - -impl AsyncWal { - fn submit(&self, data: Vec) -> WalTicket { - let ticket = WalTicket::new(); - self.tx.send(WalEntry { data, ticket: ticket.clone() }); - ticket // Caller can wait on ticket - } -} - -// Writer thread coalesces multiple writes into single fsync -fn wal_writer_loop(rx: Receiver, config: WalConfig) { - let mut buffer = Vec::new(); - loop { - match rx.recv_timeout(config.max_coalesce_delay) { - Ok(entry) => buffer.push(entry), - Err(Timeout) => { - if !buffer.is_empty() { - flush_buffer(&mut buffer); // Single fsync - } - } - } - if buffer.len() >= config.max_coalesce_records { - flush_buffer(&mut buffer); - } - } -} -``` - -### Partitioned Processing - -For non-persistent shards, records can be partitioned by identity key hash for -parallel processing. Each partition uses `process_batch_optimized()`: - -``` - ┌──────────────────────┐ - │ Incoming Records │ - └──────────┬───────────┘ - │ - ┌──────────▼───────────┐ - │ Partition by Hash │ (identity_key_hash % partition_count) - └──────────┬───────────┘ - │ - ┌────────────────────┼────────────────────┐ - │ │ │ - ┌──────▼──────┐ ┌──────▼──────┐ ┌──────▼──────┐ - │ Partition 0 │ │ Partition 1 │ │ Partition N │ - │ │ │ │ │ │ - │ 1. Batch │ │ 1. Batch │ │ 1. Batch │ - │ Insert │ │ Insert │ │ Insert │ - │ 2. Parallel │ │ 2. Parallel │ │ 2. Parallel │ - │ Extract │ │ Extract │ │ Extract │ - │ 3. Seq Link │ │ 3. Seq Link │ │ 3. Seq Link │ - └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ - │ │ │ - └────────────────────┼────────────────────┘ - │ - ┌──────────▼───────────┐ - │ Merge & Sort by │ - │ Original Index │ - └──────────────────────┘ -``` - -Each partition runs independently with its own `Mutex`. Rayon processes all partitions in parallel—no global lock contention. - -Partition stores are currently in-memory. A shard configured with `data_dir` -therefore does not use this path: it routes every batch through the shard's -`PersistentStore`, where records, indexes, and cluster assignments are durable -and immediately available to queries. Enabling durable partition-local stores -requires an explicit on-disk layout and recovery protocol; routing persistent -traffic to in-memory partitions is not a valid optimization. - ---- - -## Data Flow - -### Ingest Path - -``` -1. Client sends RecordInput batch via gRPC -2. Router hashes identity keys → partition records to shards -3. Each shard receives its partition of the batch -4. Non-persistent shards may route large batches to ParallelPartitionedUnirust -5. On that path, records are partitioned again by identity_key_hash % partition_count -6. Each partition (in parallel via Rayon): - a. Batch insert all records to store - b. Parallel extract: identity keys + strong ID summaries (Rayon) - c. Sequential link: DSU merges with temporal guards - d. Update identity index -7. Persistent shards instead resolve through Unirust backed by PersistentStore -8. Before resolution, write and fsync a versioned, checksummed binary ingest WAL -9. Persist records, indexes, cluster assignments, and request metadata -10. Sync the RocksDB WAL to stable storage -11. Remove the ingest WAL and fsync its parent directory -12. Update boundary indexes for cross-shard tracking -13. Return cluster assignments -``` - -If a shard stops before step 11, restart replays the ingest WAL using source -identity idempotency. If framing, length, or checksum validation fails, startup -fails closed and preserves the corrupt file for operator recovery. Cross-shard -merge redirects use the linker metadata column family and receive the same -stable-storage barrier before their RPC reports success. - -### Query Path - -``` -1. Client sends QueryEntitiesRequest -2. Router fans out to all shards (parallel) -3. Each shard: - a. Looks up descriptors in attribute index - b. Filters by temporal interval - c. Resolves cluster IDs via DSU - d. Computes golden records (conflict-free values) -4. Router aggregates results -5. Returns QueryOutcome: - - If single cluster matches: QueryMatches - - If multiple clusters claim same identity: QueryConflict -``` - -### Reconciliation Cycle - -``` -1. Router triggers reconciliation (periodic or on-demand) -2. Collect dirty boundary keys from all shards -3. For each key appearing on multiple shards: - a. Check for overlapping intervals - b. If overlap found: create merge candidates -4. Validate merges against temporal guards -5. Apply merges to each shard -6. Clear dirty keys -7. Report reconciliation stats -``` - ---- - -## Appendix: Tuning Profiles - -| Profile | Candidate Cap | Hot Key Threshold | Use Case | -|---------|---------------|-------------------|----------| -| Balanced | 2,000 | 50,000 | General purpose | -| LowLatency | 1,000 | 20,000 | Fast responses | -| HighThroughput | 4,000 | 100,000 | Batch processing | -| BulkIngest | 500 | 10,000 | Large loads; full resolution with lower candidate caps | -| MemorySaver | 500 | 5,000 | Reduced memory | -| BillionScale | 2,000 + persistent DSU | 100,000 | Huge datasets | - ---- - -## Appendix: Performance Characteristics - -### Verified Throughput (5-shard persistent cluster) - -The release audit on 2026-07-22 measured 50,598 records/sec for 10,000,000 -records, 16 streams, 5,000-record batches, and 10% overlap on an Apple M5 with -32 GB RAM. All 10,000,000 records were acknowledged with zero stream errors. -The measurement includes an `fsync` of the RocksDB WAL before each successful -batch acknowledgement. - -Earlier 280K-500K figures measured an in-memory partition path that did not -persist or expose its records through the shard's primary store. They are not -valid production baselines. Performance work must preserve full entity -resolution and acknowledge records only after durable storage. - -### Memory Usage - -| Component | Memory per Million Records | -|-----------|---------------------------| -| In-memory DSU | ~12 MB | -| Persistent DSU | ~4 MB (cached) | -| Identity Index | ~50 MB | -| Record Storage | ~100 MB (compressed) | - -### Latency - -| Operation | P50 | P99 | -|-----------|-----|-----| -| Single record ingest | 0.5ms | 2ms | -| Batch ingest (1000 records) | 10ms | 50ms | -| Point query | 0.2ms | 1ms | -| Range query (1000 results) | 5ms | 20ms | +# Unirust architecture and design + +This document describes the implementation in this repository for v0.2.0. Source +links identify the code behind each design choice. Configuration profiles and +performance-oriented module names describe mechanisms, not capacity or latency +guarantees. + +## Runtime structure + +The public Rust API is `Unirust`. A distributed deployment places a router in +front of shards, each of which owns a `Unirust` instance and, when configured with +a data directory, a `PersistentStore` backed by RocksDB. + +```mermaid +flowchart TD + Client[Client] --> Router[Router: placement, source reservations, queries] + Router --> ShardA[Persistent shard A] + Router --> ShardB[Persistent shard B] + ShardA --> EngineA[Unirust: staged ingest and local resolution] + ShardB --> EngineB[Unirust: staged ingest and local resolution] + EngineA --> DBA[(RocksDB A)] + EngineB --> DBB[(RocksDB B)] + Router --> Reconciliation[Boundary exchange and component reconciliation] + Reconciliation --> ShardA + Reconciliation --> ShardB +``` + +| Code | Responsibility | +| --- | --- | +| [`src/lib.rs`](src/lib.rs) | Public API, ingest lifecycle, backend selection, recovery, query execution | +| [`src/linker.rs`](src/linker.rs) | Candidate linking, strong-ID summaries, local membership, global redirects | +| [`src/dsu.rs`](src/dsu.rs) | Union-find backends and merge guards | +| [`src/index.rs`](src/index.rs) | Temporal identity-key extraction and in-memory/tiered candidate indexes | +| [`src/persistence.rs`](src/persistence.rs) | Durable records, indexes, string interning, metadata and checkpoints | +| [`src/distributed.rs`](src/distributed.rs) | Router/shard services, ingest WAL, replication and distributed queries | +| [`src/sharding.rs`](src/sharding.rs) | Boundary metadata and guarded component reconciliation | +| [`src/query.rs`](src/query.rs), [`src/graph.rs`](src/graph.rs) | Query intervals, golden descriptors and display keys | +| [`src/conflicts.rs`](src/conflicts.rs) | Conflict observations and summaries | +| [`proto/unirust.proto`](proto/unirust.proto) | External and internal RPC contracts | + +A persistent shard explicitly disables the optional partitioned ingest path. +`Partition` stores in [`src/partitioned.rs`](src/partitioned.rs) are in memory; +routing durable traffic through them would bypass the shard's authoritative +record store. The partitioned implementation remains a separate path available +to non-persistent shards. It is not the persistent production ingest architecture. +See `ShardNode::new_with_storage_paths` in [`distributed.rs`](src/distributed.rs). + +## Records, time and matching rules + +A record has a shard-local `RecordId`, a source identity +`(entity_type, perspective, uid)`, and descriptors. Each descriptor contains an +interned attribute ID, an interned value ID and a validity interval. Interned IDs +are local to the store/interner; distributed requests and boundary strong-ID +observations carry string values where agreement across stores is required. +See [`model.rs`](src/model.rs) and [`ontology.rs`](src/ontology.rs). + +Intervals are half-open: `[start, end)`. Valid intervals require `start < end`; +adjacent intervals do not overlap. `Interval` also provides unbounded interval +constructors, represented with the extreme `i64` endpoints. Descriptor validity +is separate from the time at which the record arrives. See +[`temporal.rs`](src/temporal.rs). + +An identity key names attributes whose values must match during a shared time +interval. Extraction coalesces equal values, intersects intervals across key +attributes, and produces complete key tuples. Missing attributes do not form a +complete identity key. Candidate buckets include the entity type, so records of +different entity types are not matched through the same bucket. The current +`Ontology::identity_keys_for_type` and `strong_identifiers_for_type` return the +configured rule lists for every entity type; they do not implement independent +per-type rule selection. Entity-specific `key_attributes` select display-key +attributes, not separate matching rules. + +Strong-ID merge guards compare the accumulated observations of both clusters. +A conflicting observation has the same perspective and attribute, a different +value, and an overlapping validity interval. Thus two different SSNs reported +by the same source during overlapping periods block a merge. Different sources +may report different values without triggering this particular guard; values +from the same source in disjoint periods can also coexist. The guard applies +to cluster histories, including records that are not the immediate matching +pair. See `build_record_summary`, `cluster_summaries_conflict` and +`would_create_conflict_in_clusters` in [`linker.rs`](src/linker.rs). + +Declared constraints and conflict reporting are separate from these merge +guards. [`conflicts.rs`](src/conflicts.rs) detects direct and indirect conflicts, +including perspective-scoped constraints, using sweep-line and atomic-interval +implementations. The selection heuristic is in +[`config/tuning.rs`](src/config/tuning.rs). Their cost depends on overlap, +cluster size and the observations produced; there is no universal linear-time +bound for conflict detection. + +## Local resolution and identity + +New records enter the linker after staging. Large `Unirust` ingest batches use +`link_records_batch_parallel_with_interner`: Rayon extracts keys and strong-ID +summaries in parallel, then linking and index insertion proceed sequentially in +record order. Smaller batches use `link_record`. Sequential DSU mutation lets +later records observe earlier merges and updated summaries. Both paths perform +entity resolution; `stream_records` omits graph construction and conflict +report generation, not matching or strong-ID guards. + +The identity index finds overlapping intervals and resolves stored candidates +to current DSU roots. Equal intervals already represented by a cluster do not +need additional tree entries. A limited tree lookup that reaches its limit is +retried without that limit. Candidate volume is not a strong-ID conflict and +does not permanently disable a key. A genuine conflicting key can still limit +cross-perspective linking. See `CandidateList` in [`index.rs`](src/index.rs) and +the two linking paths in [`linker.rs`](src/linker.rs). + +The implementation also retains accuracy-affecting work limits: + +- Key extraction keeps at most eight coalesced value/interval alternatives per + attribute before constructing combinations. +- The single-record path can apply deterministic, overlap-weighted stochastic + sampling and defer work according to candidate caps. +- Deferred reconciliation has its own comparison cap. The parallel batch path + does not apply the same sampling/deferred-cap logic as the single-record path. + +These are not guarantees of exhaustive matching or identical results for every +history, profile and batch shape. They must be considered when evaluating match +quality. The exact controls are in [`config/tuning.rs`](src/config/tuning.rs), +`extract_key_values_from_record` in [`index.rs`](src/index.rs), and +`link_record`/`reconcile_pending` in [`linker.rs`](src/linker.rs). + +Three forms of identity serve different purposes: + +| Identifier | Meaning | +| --- | --- | +| `RecordId` | A record identifier within one shard/store | +| `ClusterId` | An assignment maintained by the local linker; it is not simply a DSU root cast to an integer | +| `GlobalClusterId` | A shard ID, local anchor ID and version field, with redirects resolving aliases to a canonical global entity | + +The packed global format is `(shard_id << 48) | (version << 32) | local_id`. +Current linker-created IDs use version zero and anchor the local portion to a +record ID; local merges retain the minimum anchor. The field named `version` +is not an automatic conflict-resolution clock. Stable anchoring avoids using +allocation-order cluster numbers as durable global identities. + +The linker maintains member vectors and root aliases alongside the DSU. +Member vectors merge by size; root aliases follow the actual DSU winner. +`cluster_for_record`, `clusters_readonly` and +`global_cluster_id_for_readonly` expose authoritative membership without +replaying records. Normal and deferred local merges update membership, +strong-ID summaries, local IDs and global redirects together. + +A cross-shard canonical entity can contain several local clusters on one or +more shards. Applying a global redirect does not physically move records or +union remote records into a local DSU. Queries follow redirects and hydrate +all relevant local components. See `reconcile_global_cluster_ids`, +`apply_cross_shard_merges` and `clusters_for_global_ids` in +[`linker.rs`](src/linker.rs). + +## Durable ingest and recovery + +For a persistent shard, the main ingest path is: + +1. The router validates source identities and reserves each source record's + payload digest and target shard. Placement uses a complete configured + identity key when possible, then constraint/source-identity fallbacks. + Reservations protect retries and prevent the same source identity from + silently acquiring a different payload or destination. +2. The shard validates the batch and writes its binary ingest WAL before + dispatching work to an ingest worker. +3. `process_ingest_batch` builds records and calls `Unirust::stream_records`. + Records are staged, resolved and assigned to clusters through the shard's + primary store. +4. `PersistentStore::flush_staged_records` writes staged records, their indexes, + interner entries and record-count metadata in a RocksDB write batch. Cluster + assignments and cluster-count metadata are also written before success. +5. `PersistentStore::sync` flushes the RocksDB WAL to stable storage. The shard + clears the ingest WAL only after the ingest worker succeeds, then returns + assignments. + +These steps are implemented in [`distributed.rs`](src/distributed.rs), +[`lib.rs`](src/lib.rs) and [`persistence.rs`](src/persistence.rs). Ingest +commit tasks continue after client cancellation once accepted into the shard's +commit path. Successful acknowledgement is the durability boundary; a client +that loses its response must retry idempotently. + +The ingest WAL is distinct from RocksDB's WAL. It contains bincode payloads +with a magic value, format version, payload length and CRC32. Writing uses a +synchronized temporary file followed by rename. File removal and rename also +synchronize the containing directory on Unix; the non-Unix directory-sync +helper is currently a no-op. Corrupt input is quarantined and startup fails +instead of treating it as an empty batch. See `IngestWal` and +`decode_wal_batch` in [`distributed.rs`](src/distributed.rs). + +An identical source-identity/payload retry returns the existing record instead +of inserting and resolving a duplicate. Reusing the source identity with a +different payload is rejected. On an ingest error, `run_ingest_batch` discards +uncommitted staged records and drops derived linker/query/graph state so it +can be rebuilt. This is not a transaction that reverses already completed disk +writes or makes an entire router fan-out atomic. Shard mutation guards block +traffic when a failed durable operation may have left uncertain state. + +Persistent shard startup initializes the linker from committed records in +ascending record-ID order, in recovery batches, then replays a pending ingest +WAL through the normal ingest path. When persistent DSU or tiered-index backends +are selected, rebuildable derived state is cleared first. Cold candidate buckets +must not be mixed with a partially reconstructed DSU and strong-ID summaries. +Durable global redirects are restored separately; legacy allocation-order +redirects are handled by the stable-ID migration logic. Recovery resolves +historical strings through the persistent interner even when its caches are +small. + +Recovery batches limit temporary record materialization, not total recovery +work or total linker memory. Startup still processes the complete committed +history. See `create_streaming_linker` in [`lib.rs`](src/lib.rs), +`StreamingLinker::new_with_backends` in [`linker.rs`](src/linker.rs), and +`clear_rebuildable_linker_state` in [`persistence.rs`](src/persistence.rs). + +## Distributed reconciliation + +Placement reduces some cross-shard matching work but cannot establish complete +entity membership: records can match through different keys, and bridges can +connect previously separate entities. Shards therefore track dirty boundary +signatures, coalesced key intervals, global IDs and exact temporal strong-ID +observations. Distributed shard construction enables boundary tracking. + +The router fetches authoritative dirty keys and boundary metadata in chunks. +Candidate edges from all chunks accumulate in one reconciliation candidate set. +Before joining components, the router also fetches strong-ID observations for +the candidate canonical entities from their authoritative fragments. This +includes observations on previously reconciled components and clean keys, +which a current dirty-key page alone cannot describe. + +Component assembly checks accumulated cannot-link relationships and temporal +strong-ID observations before each union. A bridge with no strong ID cannot +join two components whose histories conflict. The resulting redirects map +members to a canonical primary chosen by global-ID ordering. Shards persist +applied redirects and conflict metadata; successfully reconciled dirty work is +cleared under the router's mutation coordination. See +`RouterNode::reconcile_dirty_keys` in [`distributed.rs`](src/distributed.rs) +and `ReconciliationCandidates::finish` / +`canonicalize_merges_with_observations` in [`sharding.rs`](src/sharding.rs). + +Reconciliation can be requested explicitly or scheduled by the adaptive +coordinator. Cross-shard membership is incomplete until the required successful +reconciliation has occurred. Failures are reported and consistency guards can +block traffic; eventual convergence requires available shards and successful +subsequent reconciliation, not just the passage of time. + +## Query execution and mastering + +A local query with an initialized linker uses the attribute/value/interval +index to find candidate records, reads their authoritative local membership, +and masters only those candidate clusters. It does not rerun entity resolution +over the full store after every ingest. Query text lookup does not allocate new +interner IDs. A `Unirust` queried before streaming initialization has a recovery/ +cache path instead; that first query can process the full store. + +Human-readable cluster keys are display labels, not canonical entity IDs. +Single-token keys can be generated using candidates alone. Composite keys use +the shortest unique token prefix across the complete local collision group; +their cache therefore uses all authoritative local clusters and is invalidated +by writes. Its rebuild can still read the full local history. See +`query_master_entities` and `query_cluster_keys_for` in [`lib.rs`](src/lib.rs), +and `cluster_keys_for_clusters` in [`graph.rs`](src/graph.rs). + +For a router with multiple shards, query execution has two phases: + +1. **Discover matching canonical entities.** Concurrent shard RPCs return + descriptor-match intervals grouped by canonical global ID. The router + coalesces intervals for each descriptor and intersects those sets across + all requested descriptors. Different predicates can therefore be satisfied + by different fragments of the same entity during overlapping periods. +2. **Hydrate complete matching entities.** The router requests the matching + global IDs from all shards in chunks. Shards return their local fragments, + including members that did not independently satisfy the query predicates. + The router masters the combined raw descriptors and clips the golden result + to the matching intervals. + +Fan-out concurrency and hydration chunk size are explicitly bounded in +[`distributed.rs`](src/distributed.rs); result size and total matching membership +are not bounded by those controls. A one-shard router uses the shard's direct +query endpoint. All participating fragment requests must succeed: shard errors +and invalid protocol/fragments are not converted to empty matches. There is no +cross-shard snapshot transaction spanning both phases; membership changes can +produce a retryable hydration error. + +Golden descriptors retain values over intervals where those values are +unambiguous. Conflicting values are trimmed over their overlapping portions; +mastering is not simply concatenating each shard's independently filtered +result. Multiple distinct canonical entities claiming the requested identity +can produce `QueryConflict`. See [`query.rs`](src/query.rs), +`golden_for_cluster` in [`graph.rs`](src/graph.rs), and +`query_global_entities` in [`distributed.rs`](src/distributed.rs). + +## Storage, memory and work limits + +RocksDB stores binary records and metadata in separate column families. These +include durable source identities, attribute/value and temporal indexes, +cluster assignments, interner mappings, source reservations, optional DSU data, +cold identity-key buckets and linker redirects. `index_identity` identifies +source records; `index_identity_keys` is the tiered linker's cold candidate +index. They serve different purposes. Serialization is bincode/protobuf and +fixed binary encodings, not JSON data files or a JSON WAL. Ontology JSON is an +external configuration format; graph JSON is an export format. Actual column +families and codecs are defined in [`persistence.rs`](src/persistence.rs). + +The optional tiered identity index has hot tree buckets, compact warm LRU +buckets and RocksDB cold buckets. Both ordinary and cached-key insertion run +capacity maintenance. A bucket is persisted before hot demotion/warm eviction, +and a read or update promotes the complete old bucket before modifying it. +This prevents a new hot fragment from hiding older temporal observations. +Failures to read, decode or persist a bucket propagate to callers. Without a +database, the index retains overflow rather than discard authoritative keys. +See `TieredIdentityKeyIndex` in [`index.rs`](src/index.rs). + +These tier and cache capacities do **not** bound total process memory. Local +member vectors, root aliases, strong-ID summaries, ID mappings, record-key +metadata and distributed boundary/redirect state still grow with the data. +The linker deliberately uses unbounded authoritative state even when a bounded +`LinkerStateConfig` is requested: that state does not yet have a safe durable +spill/read-through mechanism. Persistent DSU caches and tiered candidate buckets +do not change this limitation. `BillionScale` is a profile name, not a tested +billion-record capacity guarantee. + +Work also depends on key cardinality, descriptor alternatives, temporal history, +conflicting values, cluster sizes and reconciliation component sizes. Exact +fallback scans, composite label-cache construction, golden mastering and +complete startup replay can all be expensive. This document makes no fixed +throughput, hardware, memory-per-record or latency claim. Measurements belong +with their workload, storage mode, durability settings and recorded outputs; +throughput from the in-memory partition path is not a persistent-shard baseline. + +## Operational boundaries and future work + +Configured primary/passive-replica pairs synchronously replicate mutations and +validate compatibility and durable-state agreement. A replication divergence +blocks traffic. This is not quorum consensus, automatic leader election, +automatic failover or automatic replica bootstrap. Router/shard protocol checks +require compatible deployments; live protocol versions and durable WAL/backup +formats have separate versioning. See replication setup in +[`distributed.rs`](src/distributed.rs) and the transport configuration in +[`src/bin/unirust_shard.rs`](src/bin/unirust_shard.rs). + +The following are design gaps or future options, not shipped guarantees: + +- Durable partition-local stores with an explicit shared-query and recovery + contract before using partitioned ingest for persistent shards. +- Durable spill/read-through for all authoritative linker state before claiming + bounded total memory. +- Incremental composite-label collision metadata to avoid full label-cache + rebuilds after writes. +- Explicit exhaustive matching modes and stronger guarantees across sampling, + alternative pruning, batching and candidate-cap settings. + +Relevant executable coverage includes +[`resolution_capacity_regressions.rs`](tests/resolution_capacity_regressions.rs), +[`durable_ingest_regressions.rs`](tests/durable_ingest_regressions.rs), +[`selective_query_regressions.rs`](tests/selective_query_regressions.rs), +[`distributed_entity_regressions.rs`](tests/distributed_entity_regressions.rs), +[`linker_state_recovery.rs`](tests/linker_state_recovery.rs), +[`process_crash_recovery.rs`](tests/process_crash_recovery.rs) and +[`synchronous_replication.rs`](tests/synchronous_replication.rs). These tests +exercise specific invariants and failure cases; they do not establish universal +performance or availability guarantees. diff --git a/GPU.md b/GPU.md index 54f558f..cf6f73f 100644 --- a/GPU.md +++ b/GPU.md @@ -1,848 +1,69 @@ -# GPU Acceleration Research for Unirust - -This document analyzes GPU acceleration opportunities for the Unirust entity resolution system, drawing insights from the Sirius-DB GPU-native database project and comprehensive codebase analysis. - -## Executive Summary - -Unirust's architecture presents several high-impact GPU acceleration opportunities, particularly in batch hashing, interval matching, and cross-shard reconciliation. Conservative estimates suggest **10-50x speedups** for signature generation and **5-30x** for conflict detection at scale. The DSU (union-find) operations should remain on CPU due to sequential data dependencies. - ---- - -## Table of Contents - -1. [Background: GPU Database Systems](#background-gpu-database-systems) -2. [Unirust Architecture Analysis](#unirust-architecture-analysis) -3. [GPU Acceleration Candidates](#gpu-acceleration-candidates) -4. [Implementation Recommendations](#implementation-recommendations) -5. [Architecture Design](#architecture-design) -6. [Performance Projections](#performance-projections) -7. [References](#references) - ---- - -## Background: GPU Database Systems - -### Why GPUs for Data Processing? - -Modern GPUs offer massive parallelism that traditional CPUs cannot match: - -| Metric | CPU (AMD EPYC 9654) | GPU (NVIDIA H100) | Ratio | -|--------|---------------------|-------------------|-------| -| Cores | 96 | 16,896 CUDA cores | 176x | -| Memory Bandwidth | ~460 GB/s | 3.35 TB/s (HBM3) | 7.3x | -| FP64 TFLOPS | ~2.5 | 67 | 27x | -| Power | 360W | 700W | 1.9x | - -### Lessons from Sirius-DB - -Sirius is a GPU-native SQL engine from University of Wisconsin-Madison that achieves 7-12x speedups over DuckDB/ClickHouse. Key architectural insights: - -#### 1. GPU-Native vs Hybrid Approach - -> "Sirius treats GPUs as the primary execution engine, aiming to run the entire query plan—from scan to result—on the GPU. It differs from systems that retrofit GPU acceleration onto traditional CPU-optimized engines." - -**Application to Unirust**: Don't try to GPU-accelerate everything. Identify batch-parallelizable operations and keep sequential operations (DSU mutations) on CPU. - -#### 2. Memory Management Strategy - -Sirius divides GPU memory into two regions: -- **Data Caching**: Pre-allocated storage for frequently accessed data -- **Data Processing**: RMM pool allocator for intermediate results (hash tables, temporaries) - -**Application to Unirust**: Pre-allocate GPU memory for: -- Identity key signatures (read-heavy) -- Interval tree nodes (query-heavy) -- Conflict detection scratch space - -#### 3. Push-Based Execution Model - -Operators are stateless; data is pushed through the pipeline rather than pulled. This simplifies GPU kernel design and reduces synchronization overhead. - -**Application to Unirust**: Stream record batches to GPU, return cluster assignments and conflict flags. - -#### 4. Workloads That Benefit Most - -From Sirius benchmarks on TPC-H: -- **Join-heavy queries** (Q2-Q5, Q7-Q8, Q20-Q22): Highest GPU benefit -- **Large aggregations**: Memory bandwidth advantage -- **Scan-intensive workloads**: 3TB/s HBM vs ~400GB/s DDR5 - -**Application to Unirust**: Interval matching is essentially a temporal join. Cross-shard reconciliation involves large aggregations. - ---- - -## Unirust Architecture Analysis - -### Current Compute Hotspots - -Based on profiling and code analysis, the following operations dominate compute time: - -#### 1. Identity Key Signature Generation (`src/sharding.rs`) - -```rust -pub fn compute_identity_key_signature( - entity_type: &str, - key_values: &[String], -) -> IdentityKeySignature { - let mut sig = [0u8; 32]; - - // First hasher: bytes 0-15 - let mut hasher1 = DefaultHasher::new(); - entity_type.hash(&mut hasher1); - for value in key_values { - value.hash(&mut hasher1); - } - let hash1 = hasher1.finish(); - sig[0..8].copy_from_slice(&hash1.to_le_bytes()); - - // Second hasher with XOR variations: bytes 8-31 - // ... -} -``` - -**Characteristics**: -- Called once per record during ingestion -- Called during cross-shard reconciliation (potentially millions of times) -- Pure computation, no dependencies between records -- **Ideal for GPU**: Embarrassingly parallel - -#### 2. Interval Tree Queries (`src/index.rs`) - -```rust -impl IntervalTree { - pub fn collect_overlapping_limited( - &mut self, - interval: Interval, - max_nodes: usize, - out: &mut Vec<(RecordId, Interval)>, - ) -> bool { - // Binary search + traversal: O(log n + k) - // k = number of overlapping intervals - } -} -``` - -**Characteristics**: -- Called during candidate collection for each identity key match -- Tree traversal has good locality but random access patterns -- Batch queries against same tree can be parallelized -- **GPU potential**: Moderate (need to flatten tree structure) - -#### 3. DSU (Union-Find) Operations (`src/dsu.rs`) - -```rust -impl TemporalDSU { - pub fn find(&mut self, record_id: RecordId) -> RecordId { - // Path halving with root caching - if let Some(&cached_root) = self.root_cache.get(&record_id) { - return cached_root; - } - let root = self.find_root_with_path_halving(record_id, initial_parent); - self.root_cache.put(record_id, root); - root - } - - pub fn union(&mut self, a: RecordId, b: RecordId) -> RecordId { - // Rank-based union with temporal guard validation - } -} -``` - -**Characteristics**: -- Sequential data dependencies (path compression modifies parent pointers) -- 16KB LRU root cache provides excellent locality for streaming workloads -- Union operations must be serialized -- **NOT suitable for GPU**: Data dependencies limit parallelism - -#### 4. Cross-Shard Conflict Detection (`src/sharding.rs`) - -```rust -impl IncrementalReconciler { - pub fn find_cross_shard_merges_with_stats(&self) -> ReconcileResult { - // Build HashMap of all signatures across shards - let mut all_entries: HashMap> = HashMap::new(); - - // O(m²) pairwise comparison per signature - for (sig, entries) in all_entries { - for i in 0..entries.len() { - for j in (i + 1)..entries.len() { - if entries[i].shard_id != entries[j].shard_id { - if is_temporally_overlapping(entries[i], entries[j]) { - check_for_conflicts_or_merges(...); - } - } - } - } - } - } -} -``` - -**Characteristics**: -- Quadratic complexity per identity key signature -- Independent comparisons (no data dependencies) -- Memory-bound (loading boundary entries) -- **Excellent for GPU**: All-pairs comparison is classic GPU workload - -#### 5. Strong ID Summary Computation (`src/linker.rs`) - -```rust -struct StrongIdSummary { - by_perspective: HashMap>>>, -} - -impl StrongIdSummary { - fn compute_perspective_strong_ids(&self, interner: &StringInterner) -> HashMap { - // Hash perspective -> strong ID values for cross-shard comparison - let mut result = HashMap::new(); - for (perspective, attrs) in &self.by_perspective { - let perspective_hash = hash(perspective); - let values_hash = hash_all_values(attrs, interner); - result.insert(perspective_hash, values_hash); - } - result - } -} -``` - -**Characteristics**: -- Called during boundary tracking (on merge operations) -- Nested HashMap iteration -- String interning requires CPU access -- **Moderate GPU potential**: Batch computation possible after data preparation - ---- - -## GPU Acceleration Candidates - -### Tier 1: High Impact, Low Effort - -#### 1.1 Batch Identity Key Signature Generation - -**Current State**: Sequential hashing with DefaultHasher -**GPU Approach**: Parallel FxHash-style computation - -```cuda -__global__ void batch_identity_signatures( - const char* __restrict__ entity_types, // Packed strings - const uint32_t* __restrict__ type_offsets, // String boundaries - const char* __restrict__ key_values, // Packed key values - const uint32_t* __restrict__ kv_offsets, // Per-record boundaries - uint8_t* __restrict__ signatures, // Output: 32 bytes per record - uint32_t record_count -) { - uint32_t idx = blockIdx.x * blockDim.x + threadIdx.x; - if (idx >= record_count) return; - - // FxHash constants - const uint64_t K = 0x517cc1b727220a95ULL; - - uint64_t state1 = 0, state2 = 0x9e3779b97f4a7c15ULL; - - // Hash entity type - uint32_t type_start = type_offsets[idx]; - uint32_t type_end = type_offsets[idx + 1]; - for (uint32_t i = type_start; i < type_end; i++) { - state1 = (state1 ^ entity_types[i]) * K; - } - - // Hash key values - uint32_t kv_start = kv_offsets[idx]; - uint32_t kv_end = kv_offsets[idx + 1]; - for (uint32_t i = kv_start; i < kv_end; i++) { - state1 = (state1 ^ key_values[i]) * K; - state2 = (state2 ^ key_values[i]) * K; - } - - // Write 32-byte signature - uint8_t* out = signatures + idx * 32; - *((uint64_t*)(out + 0)) = state1; - *((uint64_t*)(out + 8)) = state2; - *((uint64_t*)(out + 16)) = state1 ^ state2; - *((uint64_t*)(out + 24)) = state1 * state2; -} -``` - -**Expected Speedup**: 10-50x for 100K+ records -**Implementation Effort**: ~100 lines CUDA + ~200 lines Rust bindings - -#### 1.2 Record-to-Shard Partitioning - -**Current State**: Sequential hash + modulo - -```rust -fn shard_for_record(record: &Record, ontology: &Ontology, shard_count: usize) -> usize { - let key_values = extract_key_values(record, ontology); - let hash = compute_hash(&record.identity.entity_type, &key_values); - (hash as usize) % shard_count -} -``` - -**GPU Approach**: Batch partitioning with histogram - -```cuda -__global__ void batch_partition_records( - const uint64_t* __restrict__ hashes, // Pre-computed signatures - uint32_t* __restrict__ shard_ids, // Output - uint32_t record_count, - uint32_t shard_count -) { - uint32_t idx = blockIdx.x * blockDim.x + threadIdx.x; - if (idx >= record_count) return; - - shard_ids[idx] = hashes[idx] % shard_count; -} - -// Optional: Compute per-shard histograms for load balancing -__global__ void partition_histogram( - const uint32_t* __restrict__ shard_ids, - uint32_t* __restrict__ histogram, // [shard_count] - uint32_t record_count -) { - // Shared memory histogram per block, then atomic add - __shared__ uint32_t local_hist[MAX_SHARDS]; - // ... -} -``` - -**Expected Speedup**: 20-100x for batch ingestion -**Implementation Effort**: ~50 lines CUDA - -### Tier 2: High Impact, Medium Effort - -#### 2.1 Batch Interval Overlap Queries - -**Current State**: Sequential tree traversal per query - -**GPU Approach**: Flatten interval tree to sorted array, parallel binary search - -```cuda -// Interval structure (GPU-friendly layout) -struct GPUInterval { - int64_t start; - int64_t end; - uint32_t record_id; - uint32_t padding; // Alignment -}; - -__global__ void batch_interval_overlap( - const GPUInterval* __restrict__ tree, // Sorted by start time - uint32_t tree_size, - const GPUInterval* __restrict__ queries, - uint32_t query_count, - uint32_t* __restrict__ candidate_counts, // Per-query count - uint32_t* __restrict__ candidates, // Flattened output - uint32_t max_candidates_per_query -) { - uint32_t qidx = blockIdx.x * blockDim.x + threadIdx.x; - if (qidx >= query_count) return; - - GPUInterval query = queries[qidx]; - - // Binary search for first potential overlap - uint32_t left = 0, right = tree_size; - while (left < right) { - uint32_t mid = (left + right) / 2; - if (tree[mid].end <= query.start) { - left = mid + 1; - } else { - right = mid; - } - } - - // Collect overlapping intervals - uint32_t count = 0; - uint32_t* out = candidates + qidx * max_candidates_per_query; - - for (uint32_t i = left; i < tree_size && tree[i].start < query.end; i++) { - if (tree[i].end > query.start) { // Overlap condition - if (count < max_candidates_per_query) { - out[count++] = tree[i].record_id; - } - } - } - - candidate_counts[qidx] = count; -} -``` - -**Expected Speedup**: 5-20x for 10K+ queries -**Implementation Effort**: ~200 lines CUDA + tree flattening logic - -#### 2.2 Cross-Shard Conflict Detection - -**Current State**: Nested loops with O(m²) comparisons - -**GPU Approach**: Parallel all-pairs comparison with conflict matrix - -```cuda -// Boundary entry structure -struct GPUBoundaryEntry { - uint32_t shard_id; - uint32_t cluster_id; - int64_t interval_start; - int64_t interval_end; - uint64_t perspective_hash; - uint64_t strong_id_hash; -}; - -__global__ void detect_cross_shard_conflicts( - const GPUBoundaryEntry* __restrict__ entries, - uint32_t entry_count, - uint8_t* __restrict__ conflict_matrix, // entry_count x entry_count bits - uint32_t* __restrict__ merge_candidates, // Pairs that can merge - uint32_t* __restrict__ merge_count -) { - // 2D grid: each thread handles one (i, j) pair - uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; - uint32_t j = blockIdx.y * blockDim.y + threadIdx.y; - - if (i >= entry_count || j >= entry_count || i >= j) return; - - GPUBoundaryEntry e1 = entries[i]; - GPUBoundaryEntry e2 = entries[j]; - - // Skip same-shard pairs - if (e1.shard_id == e2.shard_id) return; - - // Check temporal overlap - bool overlaps = (e1.interval_start < e2.interval_end) && - (e2.interval_start < e1.interval_end); - if (!overlaps) return; - - // Check for conflict (same perspective, different strong IDs) - bool same_perspective = (e1.perspective_hash == e2.perspective_hash); - bool different_strong_ids = (e1.strong_id_hash != e2.strong_id_hash); - - if (same_perspective && different_strong_ids) { - // Mark conflict - uint32_t bit_idx = i * entry_count + j; - atomicOr(&conflict_matrix[bit_idx / 8], 1 << (bit_idx % 8)); - } else if (!same_perspective || !different_strong_ids) { - // Potential merge candidate - uint32_t slot = atomicAdd(merge_count, 1); - merge_candidates[slot * 2] = i; - merge_candidates[slot * 2 + 1] = j; - } -} -``` - -**Expected Speedup**: 8-30x for 10K+ entries -**Implementation Effort**: ~300 lines CUDA + CPU coordination - -### Tier 3: Moderate Impact, High Effort - -#### 3.1 Strong ID Summary Aggregation - -**Challenge**: Nested HashMap structure doesn't map well to GPU - -**Approach**: Flatten to sorted arrays, GPU-side grouping - -```cuda -// Flattened strong ID entry -struct FlatStrongIdEntry { - uint32_t cluster_id; - uint64_t perspective_hash; - uint64_t attr_hash; - uint64_t value_hash; - int64_t interval_start; - int64_t interval_end; -}; - -// Step 1: Sort by (cluster_id, perspective_hash) -// Step 2: Parallel reduction to compute per-perspective hash -// Step 3: Compact results - -__global__ void compute_perspective_hashes( - const FlatStrongIdEntry* __restrict__ sorted_entries, - const uint32_t* __restrict__ group_boundaries, // Per-cluster-perspective - uint32_t group_count, - uint64_t* __restrict__ output_hashes -) { - uint32_t gidx = blockIdx.x * blockDim.x + threadIdx.x; - if (gidx >= group_count) return; - - uint32_t start = group_boundaries[gidx]; - uint32_t end = group_boundaries[gidx + 1]; - - // XOR-combine all (attr, value) hashes in group - uint64_t combined = 0; - for (uint32_t i = start; i < end; i++) { - combined ^= sorted_entries[i].attr_hash; - combined ^= sorted_entries[i].value_hash; - combined = combined * 0x517cc1b727220a95ULL; - } - - output_hashes[gidx] = combined; -} -``` - -**Expected Speedup**: 3-8x -**Implementation Effort**: ~400 lines CUDA + significant data transformation - ---- - -## Implementation Recommendations - -### Phase 1: Foundation (Week 1-2) - -1. **Add CUDA build infrastructure** - ```toml - # Cargo.toml - [features] - cuda = ["cudarc", "half"] - - [dependencies] - cudarc = { version = "0.12", optional = true } - ``` - -2. **Create GPU abstraction layer** - ```rust - // src/gpu/mod.rs - #[cfg(feature = "cuda")] - pub mod cuda; - - pub trait GpuBackend { - fn batch_signatures(&self, records: &[RecordBatch]) -> Vec; - fn batch_partition(&self, signatures: &[IdentityKeySignature], shards: u32) -> Vec; - } - ``` - -3. **Implement batch signature generation** - - Start with cudarc for kernel management - - Benchmark against current AVX2 implementation - -### Phase 2: Interval Matching (Week 3-4) - -1. **Add interval tree GPU serialization** - ```rust - impl IntervalTree { - pub fn to_gpu_format(&self) -> GpuIntervalArray { - // Flatten tree to sorted array - } - } - ``` - -2. **Implement batch overlap kernel** - -3. **Add CPU fallback for small queries** - ```rust - const GPU_QUERY_THRESHOLD: usize = 1000; - - fn batch_overlap_queries(&self, queries: &[Interval]) -> Vec> { - if queries.len() < GPU_QUERY_THRESHOLD || !self.gpu.is_available() { - return self.cpu_batch_overlap(queries); - } - self.gpu.batch_overlap(queries) - } - ``` - -### Phase 3: Cross-Shard Reconciliation (Week 5-6) - -1. **Serialize boundary entries to GPU format** -2. **Implement conflict detection kernel** -3. **Integrate with `IncrementalReconciler`** - -### Phase 4: Optimization & Integration (Week 7-8) - -1. **Memory pool management** (RMM-style) -2. **Async transfer pipelining** -3. **Fallback path testing** -4. **Benchmarking & tuning** - ---- - -## Architecture Design - -### System Overview - -``` -┌─────────────────────────────────────────────────────────────────────────┐ -│ CPU Domain │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────────┐ │ -│ │ Record │ │ DSU │ │ Ontology │ │ Persistence │ │ -│ │ Parsing │ │ Operations │ │ Config │ │ (RocksDB) │ │ -│ └──────┬──────┘ └──────▲──────┘ └─────────────┘ └─────────────────┘ │ -│ │ │ │ -│ │ RecordBatch │ ClusterAssignments │ -│ ▼ │ │ -│ ┌──────────────────────────────────────────────────────────────────┐ │ -│ │ GPU Dispatch Layer │ │ -│ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────────┐ │ │ -│ │ │ Threshold │ │ Memory │ │ Async Transfer │ │ │ -│ │ │ Check │ │ Pool │ │ Pipeline │ │ │ -│ │ └─────────────┘ └─────────────┘ └─────────────────────────┘ │ │ -│ └──────────────────────────┬───────────────────────────────────────┘ │ -└─────────────────────────────┼───────────────────────────────────────────┘ - │ PCIe / NVLink -┌─────────────────────────────▼───────────────────────────────────────────┐ -│ GPU Domain │ -│ ┌─────────────────────────────────────────────────────────────────┐ │ -│ │ Memory Regions │ │ -│ │ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │ │ -│ │ │ Signature │ │ Interval │ │ Scratch │ │ │ -│ │ │ Buffer (R/W) │ │ Tree (RO) │ │ Space │ │ │ -│ │ │ ~100MB │ │ ~500MB │ │ ~200MB │ │ │ -│ │ └─────────────────┘ └─────────────────┘ └─────────────────┘ │ │ -│ └─────────────────────────────────────────────────────────────────┘ │ -│ │ -│ ┌─────────────────────────────────────────────────────────────────┐ │ -│ │ Kernels │ │ -│ │ ┌───────────────┐ ┌────────────────┐ ┌────────────────────┐ │ │ -│ │ │ batch_hash │ │ interval_match │ │ conflict_detect │ │ │ -│ │ │ (10-50x) │ │ (5-20x) │ │ (8-30x) │ │ │ -│ │ └───────────────┘ └────────────────┘ └────────────────────┘ │ │ -│ └─────────────────────────────────────────────────────────────────┘ │ -└─────────────────────────────────────────────────────────────────────────┘ -``` - -### Data Flow - -``` -Ingestion Path: - Records → [CPU] Parse → [GPU] Batch Hash → [GPU] Partition → [CPU] DSU Link - │ - ▼ - IdentityKeySignatures - │ - ▼ - [GPU] Interval Match (candidates) - │ - ▼ - [CPU] DSU Union (sequential) - -Reconciliation Path: - BoundaryMetadata → [GPU] Conflict Detect → [CPU] Apply Merges - │ │ - ▼ ▼ - [GPU] Signature Compute Merge Candidates + Conflicts -``` - -### Memory Layout - -``` -GPU Memory Map (8GB example): -┌──────────────────────────────────────────────────────────────┐ -│ 0x0000_0000 - 0x1000_0000: Signature Buffer (256MB) │ -│ - 8M signatures × 32 bytes = 256MB │ -│ - Double-buffered for async transfer │ -├──────────────────────────────────────────────────────────────┤ -│ 0x1000_0000 - 0x3000_0000: Interval Tree (512MB) │ -│ - 32M intervals × 16 bytes = 512MB │ -│ - Read-only after initial load │ -├──────────────────────────────────────────────────────────────┤ -│ 0x3000_0000 - 0x4000_0000: Boundary Entries (256MB) │ -│ - 4M entries × 64 bytes = 256MB │ -│ - Loaded per reconciliation cycle │ -├──────────────────────────────────────────────────────────────┤ -│ 0x4000_0000 - 0x5000_0000: Scratch Space (256MB) │ -│ - Candidate lists, conflict matrix, temporaries │ -├──────────────────────────────────────────────────────────────┤ -│ 0x5000_0000 - 0x8000_0000: RMM Pool (768MB) │ -│ - Dynamic allocations │ -└──────────────────────────────────────────────────────────────┘ -``` - ---- - -## Performance Projections - -### Benchmarking Methodology - -All projections based on: -- GPU: NVIDIA A100 40GB (or equivalent) -- CPU: AMD EPYC 7763 (64 cores) -- Memory: 512GB DDR4-3200 -- Storage: NVMe SSD (7GB/s read) - -### Projected Speedups - -| Operation | Current (CPU) | Projected (GPU) | Speedup | Break-even | -|-----------|---------------|-----------------|---------|------------| -| Signature Generation (1M records) | 850ms | 17ms | **50x** | 10K records | -| Shard Partitioning (1M records) | 120ms | 2ms | **60x** | 5K records | -| Interval Queries (100K queries) | 2.1s | 140ms | **15x** | 1K queries | -| Conflict Detection (100K entries) | 4.5s | 180ms | **25x** | 5K entries | -| End-to-end Ingestion (1M records) | 12s | 4s | **3x** | N/A | - -### Cost-Benefit Analysis - -| Scenario | Records/sec (CPU) | Records/sec (GPU) | GPU Cost | ROI | -|----------|-------------------|-------------------|----------|-----| -| Small (10K rec/batch) | 50K | 80K | $3/hr | Negative | -| Medium (100K rec/batch) | 45K | 180K | $3/hr | 2-3 months | -| Large (1M rec/batch) | 40K | 400K | $3/hr | < 1 month | - -**Recommendation**: GPU acceleration is cost-effective for batches > 50K records or sustained throughput > 100K records/second. - ---- - -## Technical Considerations - -### PCIe Transfer Overhead - -PCIe 4.0 x16: ~25 GB/s theoretical, ~20 GB/s practical - -| Data Size | Transfer Time | Compute Time (GPU) | Transfer % | -|-----------|---------------|---------------------|------------| -| 10 MB | 0.5ms | 0.2ms | 71% | -| 100 MB | 5ms | 2ms | 71% | -| 1 GB | 50ms | 15ms | 77% | - -**Mitigation Strategies**: -1. Double-buffering (overlap transfer and compute) -2. Compression (2-4x for string data) -3. Batch coalescing (accumulate before transfer) - -### Graceful Degradation - -```rust -pub struct GpuAccelerator { - device: Option, - fallback_enabled: bool, -} - -impl GpuAccelerator { - pub fn batch_signatures(&self, records: &[RecordBatch]) -> Vec { - match &self.device { - Some(gpu) if records.len() > GPU_THRESHOLD => { - match gpu.batch_signatures(records) { - Ok(sigs) => return sigs, - Err(e) => { - tracing::warn!("GPU signature failed, falling back to CPU: {}", e); - } - } - } - _ => {} - } - - // CPU fallback (existing AVX2 implementation) - cpu_batch_signatures(records) - } -} -``` - -### Multi-GPU Scaling - -For distributed deployments with multiple GPUs: - -``` -Shard 0 ←→ GPU 0 Shard 1 ←→ GPU 1 Shard 2 ←→ GPU 2 - │ │ │ - └───────────────────┼────────────────────┘ - │ - NVLink/NVSwitch - (600 GB/s) - │ - Cross-shard - reconciliation -``` - ---- - -## What NOT to GPU-Accelerate - -### DSU (Union-Find) Operations - -**Why not**: -1. **Sequential dependencies**: Path compression modifies parent pointers -2. **Excellent CPU cache performance**: 16KB LRU root cache -3. **Low arithmetic intensity**: Memory-bound, not compute-bound - -**Evidence**: GPU union-find implementations achieve only 1.5-3x speedup with significantly higher complexity. - -### Single-Record Operations - -**Why not**: -1. PCIe transfer overhead dominates -2. Kernel launch latency (~10μs) exceeds computation time -3. No batching opportunity - -### String Interning - -**Why not**: -1. Requires global hash table with atomic operations -2. High collision rate degrades GPU performance -3. CPU hash tables are highly optimized - ---- - -## References - -### Sirius-DB Research - -1. **Rethinking Analytical Processing in the GPU Era** (2025) - - arXiv: https://arxiv.org/abs/2508.04701 - - Key insight: GPU-native design outperforms hybrid approaches - -2. **GPU Database Systems Characterization and Optimization** (VLDB 2024) - - https://dl.acm.org/doi/abs/10.14778/3632093.3632107 - - Bottleneck analysis for GPU databases - -3. **Tile-based Lightweight Integer Compression in GPU** (SIGMOD 2022) - - Compression techniques for GPU memory efficiency - -4. **A Study of the Fundamental Performance Characteristics of GPUs and CPUs for Database Analytics** (SIGMOD 2020) - - CPU vs GPU tradeoff analysis - -### GPU Programming Resources - -5. **CUDA C++ Programming Guide** - - https://docs.nvidia.com/cuda/cuda-c-programming-guide/ - -6. **cuDF (RAPIDS)** - - https://github.com/rapidsai/cudf - - GPU DataFrame library used by Sirius - -7. **RMM (RAPIDS Memory Manager)** - - https://github.com/rapidsai/rmm - - Pool allocator for GPU memory - -### Related Systems - -8. **HeavyDB (formerly OmniSci)** - - Open-source GPU database - - https://github.com/heavyai/heavydb - -9. **BlazingSQL** - - GPU-accelerated SQL on Apache Arrow - - Now part of RAPIDS - ---- - -## Appendix: Kernel Launch Configuration - -### Signature Generation - -```cuda -// 256 threads per block, 1 record per thread -dim3 block(256); -dim3 grid((record_count + 255) / 256); -batch_identity_signatures<<>>(args...); -``` - -### Interval Overlap - -```cuda -// 128 threads per block (memory-bound) -// Shared memory for candidate aggregation -dim3 block(128); -dim3 grid((query_count + 127) / 128); -size_t shared_mem = 128 * MAX_CANDIDATES * sizeof(uint32_t); -batch_interval_overlap<<>>(args...); -``` - -### Conflict Detection - -```cuda -// 2D grid for all-pairs comparison -// 16x16 thread blocks -dim3 block(16, 16); -dim3 grid((entry_count + 15) / 16, (entry_count + 15) / 16); -detect_cross_shard_conflicts<<>>(args...); -``` - ---- - -*Document generated: 2025-01-04* -*Last updated: 2025-01-05 (baseline updated to 410K rec/sec after batch-parallel optimization)* +# GPU acceleration: unimplemented feasibility notes + +Unirust v0.2.0 has no GPU backend, GPU feature flag or GPU build dependency. +The production ingestion and query paths execute on the CPU. This document +records possible experiments; it does not describe shipped functionality. +No GPU throughput, latency, speedup or cost measurements are available for +Unirust. Earlier numerical projections in this file were unsupported and have +been removed. + +## Current execution path + +Persistent shards call `Unirust::stream_records`. Large batches use Rayon for +identity-key extraction and strong-ID summary preparation, followed by +sequential linking and DSU mutation. Durable commit follows resolution. See +[the implementation design](DESIGN.md), [the linker](src/linker.rs) and +[the shard service](src/distributed.rs). + +[IdentityKeySignature](src/sharding.rs) is a 32-byte SHA-256 digest, not an +FNV hash or a GPU-specific value. Its numeric and text constructors use separate +versioned domain prefixes and length-prefixed fields. Numeric interner IDs are +local to a store; distributed agreement must use the appropriate text-based +representation. Any accelerated signature implementation would need to match +these bytes exactly, including ordering and encoding. + +The helpers under [src/perf](src/perf/mod.rs) are separate building blocks. +Their names do not establish that production ingestion uses SIMD hashing, +lock-free union-find, asynchronous WAL acknowledgement or GPU execution. +The authoritative production WAL and durability path are in +[distributed.rs](src/distributed.rs) and [persistence.rs](src/persistence.rs). + +## Experiments that could be evaluated + +| Candidate | Work that might be batched | Required boundary | +| --- | --- | --- | +| Key preparation | Encoding or hashing many independent keys | Preserve exact signature bytes and complete temporal key tuples | +| Interval candidate filtering | Comparing immutable interval arrays | Preserve half-open interval semantics, extreme endpoints and all required candidates | +| Reconciliation preparation | Grouping boundary signatures | Leave authoritative component-wide strong-ID checks and merge application intact | + +These are hypotheses about parallel work, not recommendations to add a particular +GPU library or buy hardware. Profiling must first establish whether each operation +materially contributes to end-to-end time. Faster key hashing would not by itself +remove durable writes, RPC latency, record hydration or sequential merge work. + +A prototype must return enough information for the existing resolution path to +perform its checks. It cannot acknowledge ingestion before durable commit, skip +resolution, or replace component-wide guards with pairwise checks. Current +candidate pruning and sampling limits are described in [DESIGN.md](DESIGN.md); +acceleration must not silently introduce further omissions. + +## Evidence required before adoption + +1. Measure the CPU reference on persistent shards with a recorded commit, + hardware, profile, ontology, topology, dataset size, overlap and seed. Include + repeated runs, failure counts, acknowledged records and RPC latency. +2. Separate key preparation, transfer, kernel, synchronization, linking and + persistence costs. Include initialization and small batches as well as large + batches; report both memory use and end-to-end results. +3. Compare cluster membership, strong-ID rejections and query results against + the CPU reference. Cover transitive merges, duplicate source identities, + different perspectives, adjacent and unbounded intervals, hot keys and + cross-shard reconciliation. +4. Exercise allocation and device failures, partial batches and restart. Verify + that fallback preserves the existing staging, error propagation and durability + contract without resolving or committing a record twice. +5. Keep a CPU-only build and deployment path. Add hardware-specific CI only once + a concrete implementation and test environment exist. + +An implementation decision should follow those measurements. No speedup target, +batch-size threshold or memory-capacity guarantee is established by this note. diff --git a/README.md b/README.md index 538ae49..2ff6733 100644 --- a/README.md +++ b/README.md @@ -4,116 +4,277 @@ Unirust Logo -A high-performance temporal entity resolution engine in Rust. +A temporal entity resolution engine in Rust, with RocksDB persistence and a +gRPC router/shard deployment model. + +The current release is [v0.2.0](https://github.com/script3r/unirust/releases/tag/v0.2.0). +The subsequent [CI fix](https://github.com/script3r/unirust/pull/8) authenticates +`protoc` downloads; it is not a new runtime release. Documentation and example +updates on `main` may be newer than the release tag. See [CHANGELOG.md](CHANGELOG.md) +for release scope and compatibility notes. + +## Entity Resolution and Time + +Unirust links source records using configured identity keys. Descriptors carry +integer, half-open validity intervals `[start, end)`, with `start < end`. Choose +one time unit for the dataset; the engine does not convert timestamps or infer +matching rules from attribute names. + +For example, two records that share an email identity key during `[0, 10)` can +form one entity. A role of `analyst` during `[0, 5)` and `manager` during `[5, 10)` +is a change over time, not an overlap. If different role values overlap, golden +output omits those values for the conflicting interval rather than choosing a +preferred source automatically. + +Configured strong identifiers guard merges when components contain different +values of the same strong attribute **in the same perspective during overlapping +time**. They do not impose a blanket ban on different values across sources. +Cross-shard reconciliation applies these guards to whole connected components, +including transitive merge candidates. + +Queries combine descriptors with AND over their simultaneous validity. For a +reconciled entity, the router combines matching descriptors and golden fields +across all contributing shards. Cross-shard reconciliation is asynchronous; +call `Reconcile` when a workflow needs pending boundary work processed before a +query. Its scheduling thresholds are not a guaranteed freshness deadline. + +Every new record goes through entity resolution. That does not imply exhaustive +matching: identity-key extraction limits each attribute to eight coalesced +value/interval alternatives, and some processing paths and tuning profiles cap +or sample candidates. Validate matching results with representative high +cardinality data. See [DESIGN.md](DESIGN.md) for algorithm and path-specific +limits. -## What is Entity Resolution? +## Quick Start -Entity resolution (also known as record linkage or data matching) is the process of identifying records that refer to the same real-world entity across different data sources. Unirust adds **temporal awareness** - it understands that entity attributes change over time and handles conflicts intelligently. +### Build Prerequisites -**Example**: Three records from different systems all referring to "John Doe": -- CRM: `name="John Doe", email="john@old.com"` (valid 2020-2022) -- ERP: `name="John Doe", email="john@new.com"` (valid 2022-present) -- Web: `name="John Doe", phone="555-1234"` (valid 2021-present) +Use Rust **1.88 or newer**, a C/C++ toolchain, CMake, libclang, and `protoc` on +`PATH`. RocksDB and its native dependencies are compiled during the build; +`build.rs` also generates Rust bindings from the protobuf schema. -Unirust will: -1. Cluster these as the same entity based on identity keys (name) -2. Detect the email conflict during the overlapping 2022 period -3. Produce a golden record for any point in time +On Debian/Ubuntu: -## Features +```bash +sudo apt-get update +sudo apt-get install -y build-essential cmake libclang-dev protobuf-compiler +``` -- **Temporal Awareness**: All data has validity intervals—merges and conflicts are evaluated per-time-period -- **Conflict Detection**: Automatic detection of attribute conflicts within clusters -- **Distributed**: Router + multi-shard architecture for horizontal scaling -- **Persistent**: RocksDB storage with crash recovery -- **Measured Performance**: Release baselines use persistent shards and include full entity resolution; see the benchmark below +On macOS, install Xcode Command Line Tools and the native dependencies, for +example with Homebrew: -## Quick Start +```bash +xcode-select --install +brew install cmake protobuf llvm +export LIBCLANG_PATH="$(brew --prefix llvm)/lib" +``` -### Installation +Build the release tag, including the optional load-test tool: ```bash -git clone https://github.com/script3r/unirust.git +git clone --branch v0.2.0 --depth 1 https://github.com/script3r/unirust.git cd unirust -cargo build --release +cargo build --release --locked --bins --features test-support ``` -### Single-Shard Mode (Development) +### Persistent Three-Shard Local Demo -```bash -# Start a single shard -./target/release/unirust_shard --listen 127.0.0.1:50061 --shard-id 0 --ephemeral +Run this from the repository root with ports **50060–50063 available** and no +`UNIRUST_CONFIG` or `UNIRUST_SHARD_*` / `UNIRUST_ROUTER_*` environment overrides. +It creates an explicit email-matching ontology, starts three persistent shards +and a router, waits for readiness, then runs a sample ingest and query client. +The script stops its processes on exit and preserves the printed data/log +directory for inspection. -# In another terminal, start the router -./target/release/unirust_router --listen 127.0.0.1:50060 --shards 127.0.0.1:50061 +```bash +bash <<'SH' +set -e +demo_dir="$(mktemp -d "${TMPDIR:-/tmp}/unirust-demo.XXXXXX")" +shard_pids=() +router_pid="" +cleanup() { + if [ -n "$router_pid" ]; then + kill -TERM "$router_pid" 2>/dev/null || true + wait "$router_pid" 2>/dev/null || true + fi + for pid in "${shard_pids[@]}"; do + kill -TERM "$pid" 2>/dev/null || true + done + for pid in "${shard_pids[@]}"; do + wait "$pid" 2>/dev/null || true + done +} +trap cleanup EXIT +trap 'exit 130' INT TERM + +cat > "$demo_dir/ontology.json" <<'JSON' +{ + "identity_keys": [{"name": "email_key", "attributes": ["email"]}], + "strong_identifiers": [], + "constraints": [] +} +JSON + +wait_ready() { + for attempt in $(seq 1 120); do + if ./target/release/unirust_healthcheck "$1" "$2" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + cat "$demo_dir"/*.log >&2 + return 1 +} + +for shard in 0 1 2; do + port=$((50061 + shard)) + ./target/release/unirust_shard \ + --listen "127.0.0.1:$port" --shard-id "$shard" \ + --data-dir "$demo_dir/shard-$shard" --allow-colocated-checkpoints \ + --ontology "$demo_dir/ontology.json" \ + > "$demo_dir/shard-$shard.log" 2>&1 & + shard_pids+=("$!") +done +for port in 50061 50062 50063; do + wait_ready --shard "http://127.0.0.1:$port" +done + +./target/release/unirust_router \ + --listen 127.0.0.1:50060 \ + --shards 127.0.0.1:50061,127.0.0.1:50062,127.0.0.1:50063 \ + --ontology "$demo_dir/ontology.json" \ + > "$demo_dir/router.log" 2>&1 & +router_pid="$!" +wait_ready --router http://127.0.0.1:50060 + +./target/release/unirust_client \ + --router http://127.0.0.1:50060 --ontology "$demo_dir/ontology.json" +printf 'Persistent demo data and logs: %s\n' "$demo_dir" +SH ``` -### Local Multi-Shard Cluster +The client prints assignments, an Alice email match, and a conflict response +for the shared `role=admin` query. This is a plaintext loopback demo. Its +`--allow-colocated-checkpoints` flag permits checkpoints beneath each shard's +data directory; it provides no independent copy if that volume is lost. +Production requires authenticated transport and checkpoint/replica storage in +independent failure domains. Different directory names or container volumes do +not, by themselves, establish that independence. + +### Local Cluster Script + +For a cluster that stays running, [scripts/cluster.sh](scripts/cluster.sh) +defaults to three persistent shards and `examples/loadtest-ontology.json`: ```bash -# Use the cluster script -SHARDS=5 ONTOLOGY=/etc/unirust/ontology.json ./scripts/cluster.sh start +./scripts/cluster.sh start +./scripts/cluster.sh status +./scripts/cluster.sh restart +./scripts/cluster.sh stop +``` + +`start` and `restart` preserve data; `status` checks process IDs, while startup +waits for gRPC readiness. The defaults are `cluster_data`, `cluster_backups`, and +`cluster_logs` beneath the repository, with automatic checkpoints disabled. +These local paths are not independent storage. Set `SHARDS`, `ONTOLOGY`, +`DATA_DIR`, `BACKUP_DIR`, `LOG_DIR`, and `CHECKPOINT_INTERVAL_SECS` consistently +for a different local deployment. The script is not a production TLS or +multi-host orchestrator. It builds the server and healthcheck binaries, but +not the load-test or sample client binaries. -# Restart the same persistent cluster without deleting records -SHARDS=5 ONTOLOGY=/etc/unirust/ontology.json ./scripts/cluster.sh restart +After stopping the local cluster, an explicit destructive reset is available: -# Destructive reset is separate and requires explicit confirmation +```bash UNIRUST_CONFIRM_RESET=1 ./scripts/cluster.sh reset +``` + +This deletes the configured `DATA_DIR`, not `BACKUP_DIR`. Keep shard count and +ontology fixed for an existing dataset; changing `SHARDS` is not an online +rebalance operation. + +### Using the Library + +The library can run a local engine against a persistent database. This example +is separate from the distributed deployment above; applications using the +cluster should send requests through the router. -# Or start manually: -./target/release/unirust_shard --listen 127.0.0.1:50061 --shard-id 0 --data-dir /data/shard0 --backup-dir /backup/shard0 -./target/release/unirust_shard --listen 127.0.0.1:50062 --shard-id 1 --data-dir /data/shard1 --backup-dir /backup/shard1 -./target/release/unirust_shard --listen 127.0.0.1:50063 --shard-id 2 --data-dir /data/shard2 --backup-dir /backup/shard2 -./target/release/unirust_shard --listen 127.0.0.1:50064 --shard-id 3 --data-dir /data/shard3 --backup-dir /backup/shard3 -./target/release/unirust_shard --listen 127.0.0.1:50065 --shard-id 4 --data-dir /data/shard4 --backup-dir /backup/shard4 +Add these dependencies to a Rust binary crate: -./target/release/unirust_router --listen 127.0.0.1:50060 \ - --shards 127.0.0.1:50061,127.0.0.1:50062,127.0.0.1:50063,127.0.0.1:50064,127.0.0.1:50065 +```toml +[dependencies] +unirust-rs = "0.2.0" +anyhow = "1" ``` -### Using the Library +A complete `src/main.rs` (with the same native build prerequisites): ```rust -use unirust_rs::{Unirust, PersistentStore, StreamingTuning, TuningProfile}; -use unirust_rs::ontology::{Ontology, IdentityKey, StrongIdentifier}; - -// Create ontology (matching rules) -let mut ontology = Ontology::new(); -ontology.add_identity_key(IdentityKey::new( - vec![name_attr, email_attr], - "name_email".to_string() -)); -ontology.add_strong_identifier(StrongIdentifier::new( - ssn_attr, - "ssn_unique".to_string() -)); - -// Open persistent store -let store = PersistentStore::open("/path/to/data")?; - -// Create engine with tuning profile -let tuning = StreamingTuning::from_profile(TuningProfile::HighThroughput); -let mut engine = Unirust::with_store_and_tuning(ontology, store, tuning); - -// Ingest records -let result = engine.ingest(records)?; -println!("Assigned {} records to {} clusters", - result.assignments.len(), - result.cluster_count); -println!("Detected {} conflicts", result.conflicts.len()); - -// Query entities -let matches = engine.query(&descriptors, interval)?; +use unirust_rs::ontology::{IdentityKey, StrongIdentifier}; +use unirust_rs::{ + Descriptor, Interval, Ontology, PersistentStore, QueryDescriptor, Record, + RecordId, RecordIdentity, StreamingTuning, Unirust, +}; + +fn main() -> anyhow::Result<()> { + let store = PersistentStore::open("library-data")?; + let mut ontology = Ontology::new(); + ontology.add_identity_key(IdentityKey::from_names(vec!["email"], "email_key")); + ontology.add_strong_identifier(StrongIdentifier::from_name("ssn", "ssn_guard")); + let mut engine = + Unirust::with_store_and_tuning(ontology, store, StreamingTuning::balanced()); + + let email = engine.intern_attr("email"); + let alice = engine.intern_value("alice@example.com"); + let interval = Interval::new(0, 10)?; + let records = vec![ + Record::new( + RecordId(0), + RecordIdentity::new("person".into(), "crm".into(), "alice-v1".into()), + vec![Descriptor::new(email, alice, interval)], + ), + Record::new( + RecordId(1), + RecordIdentity::new("person".into(), "erp".into(), "alice-v1".into()), + vec![Descriptor::new(email, alice, interval)], + ), + ]; + let result = engine.ingest(records)?; + println!( + "Assigned {} records; {} clusters, {} conflicts", + result.assignments.len(), result.cluster_count, result.conflicts.len() + ); + let outcome = engine.query(&[QueryDescriptor { attr: email, value: alice }], interval)?; + println!("{outcome:?}"); + engine.checkpoint()?; + Ok(()) +} ``` +`checkpoint()` flushes this engine's state; it does not create a coordinated +external cluster backup. See the [library API](https://docs.rs/unirust-rs/0.2.0/unirust_rs/) +and [examples/cluster.rs](examples/cluster.rs) for the gRPC client model. The +cluster example connects to already-running servers and installs its own +ontology, so use it with an empty cluster or one already using those exact +rules. It cannot replace the ontology of an existing load-test dataset. + ## Configuration -Unirust uses a layered configuration system: **CLI args > Environment variables > Config file > Defaults** +For supported shared settings, precedence is **CLI > environment > TOML > +defaults**. A TOML file is read only when selected with `--config PATH` or +`UNIRUST_CONFIG`; merely creating `unirust.toml` does not load it. CLI switches +such as `--ephemeral`, `--allow-colocated-checkpoints`, and +`--allow-destructive-admin` are not TOML settings. -### Config File (TOML) +Supply the same ontology and config version on every shard and router. Omitting +`--ontology` / its configuration setting loads an empty rule set, which does +not provide identity-key matching. Ontology JSON is external configuration; +record storage, ingest WALs, and backup manifests use binary formats. + +This is a deployment template: provision the named directories, ontology, +certificates, and reachable shard hosts before running it, and give each shard +its own ID and paths. ```toml -# unirust.toml profile = "high-throughput" [shard] @@ -121,6 +282,7 @@ listen = "0.0.0.0:50061" id = 0 data_dir = "/var/lib/unirust/shard-0" backup_dir = "/var/backups/unirust/shard-0" +ontology = "/etc/unirust/ontology.json" tls_cert = "/etc/unirust/tls/shard-0.crt" tls_key = "/etc/unirust/tls/shard-0.key" tls_client_ca = "/etc/unirust/tls/clients-ca.crt" @@ -128,132 +290,133 @@ tls_client_ca = "/etc/unirust/tls/clients-ca.crt" [router] listen = "0.0.0.0:50060" shards = ["https://shard-0:50061", "https://shard-1:50061", "https://shard-2:50061"] +ontology = "/etc/unirust/ontology.json" tls_cert = "/etc/unirust/tls/router.crt" tls_key = "/etc/unirust/tls/router.key" tls_client_ca = "/etc/unirust/tls/clients-ca.crt" shard_tls_ca = "/etc/unirust/tls/shards-ca.crt" shard_tls_cert = "/etc/unirust/tls/router-client.crt" shard_tls_key = "/etc/unirust/tls/router-client.key" - -[storage] -block_cache_mb = 1024 -write_buffer_mb = 256 +checkpoint_interval_secs = 3600 ``` +For example, after saving an adapted file as `/etc/unirust/unirust.toml`, start +each service on its designated host with `unirust_shard --config +/etc/unirust/unirust.toml` or `unirust_router --config /etc/unirust/unirust.toml`. +The examples below assume the installed binaries are on `PATH`; source builds +place them in `target/release/`. + ### Environment Variables -| Variable | Description | -|----------|-------------| -| `UNIRUST_CONFIG` | Path to config file | -| `UNIRUST_PROFILE` | Tuning profile | -| `UNIRUST_SHARD_LISTEN` | Shard listen address | -| `UNIRUST_SHARD_ID` | Shard ID | -| `UNIRUST_SHARD_DATA_DIR` | Persistent shard data directory | -| `UNIRUST_SHARD_BACKUP_DIR` | External checkpoint root | -| `UNIRUST_SHARD_TLS_CERT` | Shard server certificate | -| `UNIRUST_SHARD_TLS_KEY` | Shard server private key | -| `UNIRUST_SHARD_TLS_CLIENT_CA` | CA for required shard client certificates | -| `UNIRUST_SHARD_REPLICA` | Passive replica endpoint for this primary | -| `UNIRUST_SHARD_REPLICA_MODE` | Run as a passive replica | +The full mapping is in [src/config/mod.rs](src/config/mod.rs). Common settings +include: + +| Variable | Meaning | +|----------|---------| +| `UNIRUST_CONFIG` | Explicit TOML path | +| `UNIRUST_PROFILE` | Linker tuning profile | +| `UNIRUST_SHARD_LISTEN`, `UNIRUST_SHARD_ID` | Shard endpoint and logical ID | +| `UNIRUST_SHARD_DATA_DIR`, `UNIRUST_SHARD_BACKUP_DIR` | Data and checkpoint roots | +| `UNIRUST_SHARD_ONTOLOGY`, `UNIRUST_ROUTER_ONTOLOGY` | Matching-rule JSON paths | +| `UNIRUST_ROUTER_LISTEN`, `UNIRUST_ROUTER_SHARDS` | Router endpoint and comma-separated shard addresses | +| `UNIRUST_ROUTER_CHECKPOINT_INTERVAL_SECS` | Automatic checkpoint interval; `0` disables | +| `UNIRUST_ROUTER_SHARD_CONNECT_TIMEOUT_SECS` | Shard connection timeout; default 10 seconds | +| `UNIRUST_ROUTER_SHARD_REQUEST_TIMEOUT_SECS` | Per-shard RPC timeout; default 120 seconds | +| `UNIRUST_ROUTER_SHARD_TCP_KEEPALIVE_SECS` | TCP keepalive interval; default 30 seconds | +| `UNIRUST_SHARD_REPLICA`, `UNIRUST_SHARD_REPLICA_MODE` | Primary's replica endpoint, or passive mode | | `UNIRUST_SHARD_REPLICATION_TOKEN_FILE` | Shared secret file for one replica pair | -| `UNIRUST_SHARD_ALLOW_INSECURE_REPLICATION` | Permit plaintext replication for isolated development | -| `UNIRUST_SHARD_REPLICA_CONNECT_TIMEOUT_SECS` | Replica connection timeout | -| `UNIRUST_SHARD_REPLICA_REQUEST_TIMEOUT_SECS` | Per-RPC replica timeout | -| `UNIRUST_SHARD_REPLICA_TCP_KEEPALIVE_SECS` | Replica TCP keepalive interval | -| `UNIRUST_SHARD_REPLICA_TLS_CA` | CA used to verify the replica | -| `UNIRUST_SHARD_REPLICA_TLS_CERT` | Primary certificate presented to the replica | -| `UNIRUST_SHARD_REPLICA_TLS_KEY` | Primary client private key | -| `UNIRUST_ROUTER_SHARDS` | Comma-separated shard addresses | -| `UNIRUST_ROUTER_CHECKPOINT_INTERVAL_SECS` | Coordinated checkpoint interval (`0` disables) | -| `UNIRUST_ROUTER_SHARD_CONNECT_TIMEOUT_SECS` | Shard connection timeout | -| `UNIRUST_ROUTER_SHARD_REQUEST_TIMEOUT_SECS` | Per-RPC shard timeout | -| `UNIRUST_ROUTER_TLS_CERT` | Router server certificate | -| `UNIRUST_ROUTER_TLS_KEY` | Router server private key | -| `UNIRUST_ROUTER_TLS_CLIENT_CA` | CA for required router client certificates | -| `UNIRUST_ROUTER_SHARD_TLS_CA` | CA used to verify shard certificates | -| `UNIRUST_ROUTER_SHARD_TLS_CERT` | Router certificate presented to shards | -| `UNIRUST_ROUTER_SHARD_TLS_KEY` | Router client private key | - -### Tuning Profiles - -| Profile | Use Case | -|---------|----------| -| `balanced` | General purpose (default for library) | -| `low-latency` | Interactive queries, fast responses | -| `high-throughput` | Batch processing (default for binaries) | -| `bulk-ingest` | Large initial loads with lower candidate caps; entity resolution remains enabled | -| `memory-saver` | Constrained environments | -| `billion-scale` | Disk-backed DSU/index; see the recovery and memory limits below | - -## API Reference - -### gRPC Services - -**Router Service** (client-facing): -- `IngestRecords` - Ingest a batch of records -- `QueryEntities` - Query entities by descriptors and time range -- `ListConflicts` - List detected conflicts -- `GetStats` - Get cluster statistics -- `Reconcile` - Trigger cross-shard reconciliation - -**Shard Service** (internal): -- Same as router, plus boundary tracking RPCs - -### Library API +| `UNIRUST_SHARD_REPLICA_REQUEST_TIMEOUT_SECS` | Replica RPC timeout, including pairing digest | -```rust -// Core operations -engine.ingest(records) -> IngestResult -engine.query(descriptors, interval) -> QueryOutcome -engine.clusters() -> Clusters -engine.graph() -> KnowledgeGraph - -// Persistence -engine.checkpoint() -> Result<()> +Server TLS settings map to `UNIRUST_SHARD_TLS_CERT`, `_TLS_KEY`, and +`_TLS_CLIENT_CA`, or the corresponding `UNIRUST_ROUTER_*` names. Outgoing shard +TLS uses `UNIRUST_ROUTER_SHARD_TLS_CA`, `_TLS_CERT`, and `_TLS_KEY`; outgoing +replica TLS uses `UNIRUST_SHARD_REPLICA_TLS_CA`, `_TLS_CERT`, and `_TLS_KEY`. +See each binary's `--help` for supported command-line options. -// Metrics -engine.stats() -> Stats -``` +### Effective RocksDB Tuning -## Architecture +In v0.2.0, `[storage]` TOML fields and `UNIRUST_STORAGE_*` variables are parsed +by shared configuration but are **not applied when the shard opens RocksDB**. +`PersistentStore` instead reads these process environment variables directly: -See [DESIGN.md](DESIGN.md) for detailed architecture documentation, including: -- Entity resolution algorithm (4-phase streaming linker) -- Conflict detection algorithms (sweep-line vs atomic intervals) -- Distributed architecture (router + shards) -- Cross-shard reconciliation protocol -- Storage layer (RocksDB column families) -- Performance optimizations +| Variable | Actual default | Effect | +|----------|---------------:|--------| +| `UNIRUST_BLOCK_CACHE_MB` | 512 | Block cache size in MiB | +| `UNIRUST_WRITE_BUFFER_MB` | 128 | Write buffer size in MiB | +| `UNIRUST_MAX_WRITE_BUFFERS` | 4 | Maximum write buffers | +| `UNIRUST_COMPACTION_THREADS` | 1 | Background compaction threads | +| `UNIRUST_FLUSH_THREADS` | 2 | Background flush threads | +| `UNIRUST_RATE_LIMIT_MBPS` | 20 | RocksDB background I/O rate limit in MiB/s; `0` disables the limiter | -## Examples +These are per-process settings, not a total cluster memory budget. See +[src/persistence.rs](src/persistence.rs) for the remaining storage knobs. -The `examples/` directory demonstrates the supported sharded deployment model: +### Tuning Profiles -- `cluster.rs` - Full 3-shard distributed cluster with router -- `unirust.toml` - Persistent router and shard configuration - -Run examples: -```bash -# Distributed cluster (requires the persistent cluster running first) -SHARDS=3 ./scripts/cluster.sh start -cargo run --example cluster -./scripts/cluster.sh stop -``` +| Profile | Intended use | +|---------|--------------| +| `balanced` | General purpose; library default | +| `low-latency` | Lower latency tuning | +| `high-throughput` | Batch workloads; binary default | +| `bulk-ingest` | Initial loads with lower candidate caps | +| `memory-saver` | Smaller caches and candidate budgets | +| `billion-scale` | Disk-backed DSU/index with tighter memory settings | +| `billion-scale-high-performance` | Larger caches than `billion-scale` | + +Profile names are tuning presets, not latency, capacity, or matching-completeness +guarantees. Entity resolution stays enabled in every profile. The recovery and +memory limits below still apply. + +## API and Architecture + +The exact gRPC contract is [proto/unirust.proto](proto/unirust.proto). +`RouterService` offers batch ingest, entity queries, conflict listing, +statistics/metrics, health, ontology configuration, reconciliation, coordinated +checkpoints, and administrative record export/import. `IngestRecordsFromUrl` +is declared but returns `UNIMPLEMENTED`. Import is not an online shard-movement +API. `Reset` is disabled by default on shards. + +`ListConflicts` rebuilds an in-memory view from all exported records when its +cache is invalidated; it can be expensive after ingest. Router `GetStats` sums +local cluster counts, so `cluster_count` is not a count of unique reconciled +global entities. Use the semantic health RPC for readiness. + +`ShardService` is internal. It additionally exposes source-identity reservations, +entity-fragment queries, boundary metadata, merge application, and streaming +ingest. The router has no streaming-ingest RPC: clients split ingest into bounded +`IngestRecords` batches. Router record export/import also have streaming forms; +each message remains subject to size limits. + +Core library methods return: + +| Method | Return type | +|--------|-------------| +| `engine.ingest(records)` | `anyhow::Result` | +| `engine.query(&descriptors, interval)` | `anyhow::Result` | +| `engine.clusters()` | `anyhow::Result` | +| `engine.graph()` | `anyhow::Result` | +| `engine.checkpoint()` | `anyhow::Result<()>` | +| `engine.stats()` | `Stats` | + +See [DESIGN.md](DESIGN.md) for matching, temporal guards, candidate indexes, +cross-shard reconciliation, persistence, and known implementation limits. ## Durability -Persistent shards use two recovery layers for every ingest request: +Persistent shard ingest uses an application WAL and the RocksDB WAL: 1. The request is written to a versioned, checksummed binary ingest WAL. The file and its parent directory are synced before entity resolution begins. -2. Records, indexes, cluster assignments, and all other state produced by the - request are written to RocksDB, then its WAL is synced to stable storage. +2. Records, indexes, and cluster assignments are written to RocksDB, then its + WAL is synced before acknowledgement. Derived linker state can be rebuilt + from the records during recovery. 3. Only after that sync succeeds is the ingest WAL removed and the request acknowledged. Its directory is synced again after removal. On restart, a remaining ingest WAL is replayed idempotently. A pending WAL is -never overwritten by a later request; a failed ingest therefore requires shard -restart and replay before more traffic is accepted. A truncated or corrupt WAL +never overwritten by a later request. A mutation failure that leaves pending +WAL or uncertain store state requires shard restart and recovery before traffic +resumes; a validation error rejected before mutation does not imply this state. A truncated or corrupt WAL is preserved with a `.corrupt.*` suffix and shard startup fails with a data-loss error instead of accepting traffic with an unknown recovery gap. Cross-shard merge redirects are also persisted before acknowledgement and reloaded when the @@ -300,14 +463,15 @@ require a future transactional relocation protocol or an offline rebuild. Cross-shard redirects are durably applied to every shard and are idempotent. If any shard fails, or the initiating request is cancelled, while a reconciliation -result is being applied, the router latches the cluster closed for ingest, query, -and administrative traffic rather than serving a partially updated global view. +result is being applied, the router blocks ingest, entity queries, and readiness +while its global view may be partially updated. Recovery RPCs remain available. Retrying `Reconcile` repairs the retained dirty keys in place. After a router or full-cluster restart, router startup performs that repair before returning a serviceable node and clears the dirty generation only after every shard converges. -Cluster-wide ontology replacement has the same fail-closed cancellation +Changing ontology is allowed only on empty stores (reapplying the same rules +is idempotent). A cluster-wide replacement has fail-closed cancellation semantics. Readiness stays failed after an ambiguous partial update until `SetOntology` is retried with the intended configuration or every shard is recovered offline to one configuration. Router startup also rejects mismatched @@ -315,13 +479,12 @@ shard ontologies. The shard reconstructs all derived linker state before opening its gRPC listener. Recovery scans persisted records in ordered, bounded batches through -the normal entity-resolution path, so recovery time remains O(record count). -This is a correctness-first crash-recovery path, not a bounded recovery-time -guarantee. Measure restart time at the intended dataset size and set +the normal entity-resolution path. It reads the full dataset and performs the +associated matching work; recovery has no fixed duration guarantee. Measure restart time at the intended dataset size and set orchestration startup probes accordingly. -Global cluster IDs are anchored to durable record IDs so replay order cannot -change cross-shard identity. On the first startup of a database created before +Global cluster IDs use durable record anchors instead of allocation-order +local cluster IDs, avoiding allocation-order redirect drift during replay. On the first startup of a database created before this scheme marker existed, the shard atomically removes allocation-order redirects that cannot be trusted after replay; router startup reconstructs them from authoritative records before becoming ready. Upgrade every shard and the @@ -330,21 +493,15 @@ router together for this transition rather than mixing versions. `LinkerStateConfig` cache capacities are not enforced because the current LRU backend has no durable spill/read-through path. Evicting cluster IDs, strong-ID summaries, or record perspectives would change entity-resolution results. -Persistent profiles therefore retain this correctness-critical working set in -memory even though their DSU and identity index are disk-backed. The +This correctness-critical working set remains in memory even when the selected +profile uses a disk-backed DSU and identity index. The `billion-scale` profile must not be treated as proof that a billion-record deployment fits a given memory or recovery-time budget. -`scripts/cluster.sh start` and `restart` preserve `DATA_DIR`. Only the explicit -`reset` action deletes shard data, and it requires `UNIRUST_CONFIRM_RESET=1`. -The script defaults to `examples/loadtest-ontology.json`; set `ONTOLOGY` to the -same immutable configuration on every shard and router for another deployment. -Router startup compares the complete ontology reported by every shard and fails -closed on a mismatch. - The destructive gRPC `Reset` method is disabled by default because a sequential -multi-shard reset cannot be atomic. The supported production reset is the -confirmed offline script action. Test or isolated admin deployments can opt in +multi-shard reset cannot be atomic. Reset an entire deployment offline, with +all writers and shards stopped. The confirmed script action above is available +for script-managed local data. Test or isolated admin deployments can opt in with the shard flag `--allow-destructive-admin`. The shard and router binaries handle SIGINT/SIGTERM with graceful gRPC shutdown. @@ -373,21 +530,32 @@ idempotently. A logical shard can run as one primary and one passive replica on distinct persistent volumes and failure domains. Both processes use the same shard ID, -ontology, config version, and replication token. Bootstrap both volumes from -the same committed checkpoint, or start with two empty volumes. Primary startup -computes a SHA-256 digest over every logical RocksDB key/value pair on both -nodes and refuses traffic unless their complete durable states match. Pairing -startup is O(database size), so measure it against the production dataset. +ontology, config version, live protocol, and replication token. Configure the +router with primary endpoints only; it rejects passive replicas. Bootstrap +both volumes from the same committed checkpoint, or start with two empty +volumes. Primary startup +compares SHA-256 digests over the explicit `DURABLE_STATE_COLUMN_FAMILIES` +list in `src/persistence.rs`, including records, reservations, interning, indexes, +and stored linker metadata. It rejects a mismatch in that covered state; this +is not a byte-for-byte comparison of database files or every dynamically created +column family. Pairing reads all covered key/value pairs, so measure startup +against the production dataset. Set `UNIRUST_SHARD_REPLICA_REQUEST_TIMEOUT_SECS` above the measured worst-case pairing digest time. Generate a separate secret for each pair, store at least 32 random bytes in a -file readable only by the service account, and mount the same content on both -nodes. Start the passive replica first: +file readable only by the service account, and mount exactly the same file +content on both nodes. For example, `openssl rand -hex 32` produces a suitable +secret; create its destination with restrictive permissions. The token loader +hashes the complete file contents, including any trailing newline. + +The following are deployment templates, requiring the named storage, ontology, +DNS hosts, and certificates to be provisioned. Start the passive replica first: ```bash unirust_shard \ - --shard-id 0 \ + --listen 0.0.0.0:50061 --shard-id 0 \ + --ontology /etc/unirust/ontology.json \ --data-dir /var/lib/unirust/shard-0-replica \ --backup-dir /var/backups/unirust/shard-0-replica \ --replica-mode \ @@ -397,13 +565,18 @@ unirust_shard \ --tls-client-ca /etc/unirust/tls/primaries-ca.crt ``` -Then start the primary with its normal shard server credentials plus: +Then start the primary, including its server credentials and its outgoing +replica credentials: ```bash unirust_shard \ - --shard-id 0 \ + --listen 0.0.0.0:50061 --shard-id 0 \ + --ontology /etc/unirust/ontology.json \ --data-dir /var/lib/unirust/shard-0-primary \ --backup-dir /var/backups/unirust/shard-0-primary \ + --tls-cert /etc/unirust/tls/shard-0-primary.crt \ + --tls-key /etc/unirust/tls/shard-0-primary.key \ + --tls-client-ca /etc/unirust/tls/routers-ca.crt \ --replica https://shard-0-replica:50061 \ --replication-token-file /etc/unirust/replication/shard-0.token \ --replica-tls-ca /etc/unirust/tls/replicas-ca.crt \ @@ -411,10 +584,10 @@ unirust_shard \ --replica-tls-key /etc/unirust/tls/shard-0-primary.key ``` -Every durable mutation is applied to the replica first, then locally, under one +Replicated mutations are applied to the replica first, then locally, under one per-pair serialization gate. The primary acknowledges only after both results -match. An unavailable or ambiguous replica result latches the primary -unhealthy and blocks reads and writes until operators reconcile the pair. +match. Replica errors or ambiguous results fail primary readiness and block ingest +and entity queries until operators reconcile the pair. Replication therefore protects acknowledged writes from one volume loss, but adds replica latency and requires both nodes to be available for writes. @@ -422,10 +595,17 @@ Failover is manual because Unirust does not implement leader election or quorum fencing: 1. Prove the old primary is stopped or isolated from clients and the replica. -2. Stop the passive process and restart its volume without `--replica-mode`. -3. Point the router at the promoted endpoint and restart the router. -4. Rebootstrap the old primary from a checkpoint of the promoted node before - attaching it as a new passive replica. +2. Stop the passive process and restart its volume with replica mode disabled + in CLI, environment, and TOML. Preserve its ontology and provision server TLS + trust for router clients; the replica template above trusts primaries only. +3. Point the router at the promoted endpoint, configure trust for that server + certificate, and restart the router. +4. Before adding a passive replica again, ensure both pair members have identical + durable state and restore provenance. A checkpoint-based rebootstrap requires + quiescing writes, creating a fresh consistent cluster checkpoint, stopping the + cluster, and restoring every active shard and replica from that generation. + Restore both members of each pair from the same per-shard checkpoint. Restoring + only the old primary from a new generation fails provenance checks. Never serve the old primary and promoted replica simultaneously. Doing so can create split brain. Online reset is disabled while a primary has a replica; @@ -447,29 +627,46 @@ backup_dir = "/var/backups/unirust/shard-0" checkpoint_interval_secs = 3600 ``` -Trigger checkpoints through the router so router-mediated mutations remain -blocked for the complete cluster snapshot. A supplied name is created beneath -every shard's configured backup root: +Trigger checkpoints through the router. Its mutation gate blocks operations +through that router for the duration of each checkpoint call; direct shard +writes or another coordinator bypass that gate. A supplied name is created +beneath every shard's configured backup root. From the repository root, using +`grpcurl` against a local plaintext router: ```bash -grpcurl -plaintext \ +grpcurl -plaintext -import-path proto -proto unirust.proto \ -d '{"path":"backup-2026-07-24T1300Z"}' \ 127.0.0.1:50060 unirust.RouterService/Checkpoint ``` +The servers do not enable gRPC reflection, so supply the schema as shown. For +a secured router, replace `-plaintext` with `-cacert`, `-cert`, and `-key` options +pointing to your provisioned credentials. + Checkpoint creation uses a two-phase prepare/commit protocol. Every shard first flushes its in-memory linker state and creates a RocksDB snapshot. Only after all shards prepare successfully does the router write a binary commit marker to every snapshot. The response includes the shared `generation` and -`committed: true`. A failed generation remains uncommitted and cannot be -restored; retrying the same name is idempotent and completes any missing -prepare or commit steps. Do not call the shard checkpoint RPC directly for a -production backup. +`committed: true`. A failed call can leave some snapshots prepared and some +committed. Retry the same name to complete missing prepare/commit steps, and +accept a backup only when every shard has a valid committed snapshot from the +same generation. Do not call the shard checkpoint RPC directly for a production +cluster backup. + +**Retry limitation:** the router releases its mutation gate between failed +checkpoint calls. Already-prepared snapshots are reused, so writes between +attempts can make a completed generation span different points in time. Quiesce +application writes before checkpointing and keep them paused through retries +when a consistent cluster snapshot is required. If writes resumed after a +partial prepare, discard that generation as a consistency candidate and create +a fresh one while quiesced. Generation markers and checksums cannot establish +that no writes occurred between prepare attempts. The router scheduler waits one configured interval before its first checkpoint. If a shard fails during prepare or commit, the scheduler retains and retries the same immutable generation instead of creating a stream of unrelated partial -snapshots. Successful and failed generations are emitted to structured logs. +snapshots. This scheduler does not keep writers quiesced between attempts. +Successful and failed generations are emitted to structured logs. The container deployment enables hourly checkpoints by default; set `UNIRUST_CHECKPOINT_INTERVAL_SECS` explicitly to choose another RPO or `0` to disable them. @@ -479,20 +676,23 @@ shard from the same checkpoint generation into an empty replacement directory: ```bash unirust_shard \ - --shard-id 0 \ + --listen 127.0.0.1:50061 --shard-id 0 \ --data-dir /replacement/shard-0 \ --backup-dir /var/backups/unirust/shard-0 \ --restore-from /var/backups/unirust/shard-0/backup-2026-07-24T1300Z \ --ontology /etc/unirust/ontology.json ``` -Restore refuses a non-RocksDB source, symlinks, a nonempty destination, and an +Restore refuses a non-RocksDB source, symlinked sources or entries, a nonempty +destination, and an existing partial staging directory. It also requires matching binary prepare/commit markers, verifies the checkpoint belongs to the requested shard, and opens both the source and staged copy read-only with RocksDB paranoid checks. It copies and syncs into a sibling staging directory before publishing the -replacement with one rename. Restore the whole cluster together; restoring -only one older shard beside newer peers can violate the cluster snapshot +replacement with one rename. The restore command above uses loopback; apply +your normal endpoint and TLS settings when restoring each production shard. +Restore the whole cluster together; restoring only one older shard beside newer +peers can violate the cluster snapshot boundary. Each shard retains the committed checkpoint provenance in its replacement data directory and refuses a manifest for another shard. Router startup requires every shard to be either unrestored or restored from the same @@ -519,8 +719,9 @@ same generation and topology. It copies into a sibling staging directory, records every file length and SHA-256 digest in a binary manifest, opens every copied RocksDB checkpoint read-only with paranoid checks, syncs the tree, and publishes it with one rename. Verification rejects modified, missing, extra, or -symlinked content. Restore from the exported `shard-0`, `shard-1`, and -`shard-2` directories, not from the deleted local checkpoint roots. +symlinked content. To recover using an export, pass its `shard-0`, `shard-1`, +and `shard-2` directories to each corresponding shard's `--restore-from`. Export does not +delete the local source checkpoints. Retention only removes generations after every entry in its root verifies: @@ -530,11 +731,13 @@ unirust_backup prune --root /mnt/off-host/unirust --retain 14 The built-in scheduler creates coordinated source checkpoints; it does not automatically run the export command. Schedule export and verification after -checkpoint completion, monitor both, and run periodic restore drills. The -destination filesystem must provide independent storage and encryption at rest. -Without an enabled synchronous replica, the recovery point for a lost volume -remains the last successfully exported generation and acknowledged writes -after it can be lost. Process crashes and ordinary restarts remain covered +checkpoint completion, monitor both, and run periodic restore drills. Provide +independent destination storage and encryption at rest at the +infrastructure layer; the export tool does not configure either. Without a +synchronous replica, volume-loss recovery is limited to the last intact, +consistent cluster checkpoint on surviving storage, typically the last verified +off-host export. Acknowledged writes after that recovery point can be lost. +Process crashes and ordinary restarts remain covered independently by the synced ingest and RocksDB WALs. ## Deployment Security @@ -553,8 +756,10 @@ are required together. Certificate rotation currently requires a process restart. Production deployments must enable native mTLS on the router and every shard, or enforce equivalent authenticated TLS through a service mesh. Keep shard ports private and never expose plaintext gRPC to an untrusted network. -The supplied Compose file is a local plaintext example and binds its router -port to loopback only. +mTLS authenticates certificate holders; it does not provide per-user or +per-method authorization. Restrict administrative RPC access at a trusted +proxy/network boundary. The supplied Compose file is a local plaintext example +and binds its router port to loopback only. Replication additionally uses a per-pair shared token. The shard binary rejects plaintext replication by default; `--allow-insecure-replication` exists only @@ -562,105 +767,118 @@ for isolated development. Protect token files as credentials, rotate them by stopping and restarting both members, and use different tokens for every pair. The shard binary requires a persistent `--data-dir` and a non-overlapping -`--backup-dir`. Mount them from independent storage: separate directory names -alone do not protect against volume loss. An in-memory shard can only be started +`--backup-dir` unless the local-development allowance is set. The path checks +reject containment, including canonical-path overlap, but do not verify separate +filesystems or physical devices. Mount them from independent storage: separate +directory names alone do not protect against volume loss. An in-memory shard +can only be started with the explicit `--ephemeral` flag and loses all records when the process exits. `--allow-colocated-checkpoints` is a development-only escape hatch and does not provide volume-loss recovery. Router and shard servers cap each encoded or decoded gRPC message at 4 MiB, limit each connection to 128 concurrent requests, and shed excess load. Use the -streaming ingest, import, and export RPCs for larger transfers. Bound connection +router's bounded batch ingest and chunked record import/export for larger +transfers; streaming does not remove the per-message limit. Bound connection counts and request rates at the load balancer as well. -## Performance - -Release verification on an Apple M5 with 32 GB RAM, five persistent shards, -16 concurrent streams, 5,000-record batches, and 10% overlap: - -| Records | Records/sec | Stream Errors | -|---------|-------------|---------------| -| 10,000,000 | 50,598 | 0 | +## Performance and Development -This is an end-to-end power-loss-durability measurement: records and cluster -assignments are persisted and the RocksDB WAL is synchronously flushed before -acknowledgement, and every record goes through entity resolution. Results depend -on storage hardware and ontology complexity; rerun the command below on the -release target rather than treating this number as a service-level guarantee. +Measure the actual release, dataset, ontology, persistent storage, and topology +you intend to deploy. A record-throughput result is not evidence of query +latency, crash-recovery time, match completeness, or survival of physical volume +loss. This README does not claim a portable throughput or capacity guarantee. -## Development +A reproducible local load-test setup uses a fresh directory, five persistent +shards, 16 concurrent client streams, 5,000-record batches, and 10% generated +overlap. The script-managed backups are still local demo storage. With ports +50060–50065 available: ```bash -# Fast correctness gate (unit and integration tests) -cargo test - -# Compile examples, binaries, and benchmarks without executing benchmarks -cargo check --all-targets --all-features +bash <<'SH' +set -e +cargo build --release --locked --bin unirust_loadtest --features test-support +perf_dir="$(mktemp -d "${TMPDIR:-/tmp}/unirust-perf.XXXXXX")" +export SHARDS=5 +export DATA_DIR="$perf_dir/data" BACKUP_DIR="$perf_dir/backups" +export LOG_DIR="$perf_dir/logs" RUN_DIR="$perf_dir/run" +export ONTOLOGY="$PWD/examples/loadtest-ontology.json" +trap './scripts/cluster.sh stop' EXIT +./scripts/cluster.sh start +./target/release/unirust_loadtest \ + --router http://127.0.0.1:50060 --count 1000000 \ + --streams 16 --batch 5000 --overlap 0.1 --headless +printf 'Persistent performance data and logs: %s\n' "$perf_dir" +SH +``` -# Run quick benchmarks (~30s) -cargo bench --bench bench_quick +Use the same settings for comparisons; report acknowledged records, errors, +hardware, and storage configuration alongside throughput. Run benchmark suites +explicitly; their duration depends on the machine: -# Run load test (start cluster first: SHARDS=5 ./scripts/cluster.sh start) -./target/release/unirust_loadtest \ - --router http://127.0.0.1:50060 \ - --count 10000000 \ - --streams 16 \ - --batch 5000 - -# Format and lint -cargo fmt -cargo clippy --all-targets --all-features -- -D warnings +```bash +cargo bench --locked --bench bench_quick +cargo bench --locked --bench bench_distributed ``` -`cargo test --all-targets` executes Criterion benchmark binaries on some Cargo -versions and is intentionally not the default correctness gate. Run benchmarks -explicitly with `cargo bench --bench `. +The CI correctness and packaging checks are: -### Test Strategy +```bash +cargo test --locked --all-features +cargo check --locked --all-targets --all-features +cargo fmt --check +cargo clippy --locked --all-targets --all-features -- -D warnings +RUSTDOCFLAGS="-D warnings" cargo doc --locked --all-features --no-deps +cargo package --locked +``` -- Unit tests cover temporal algebra, matching, DSU behavior, indexes, and focused - persistence failure modes. -- Integration tests exercise router and shard RPCs with temporary - `PersistentStore` databases, including restart, WAL, reconciliation, streaming, - reset, and rebalance behavior. -- `cargo check --all-targets --all-features` compiles examples and benchmarks - without mixing performance workloads into the correctness suite. -- `bench_quick` is the local performance smoke test. The distributed load test is - the release benchmark and must be run against persistent shards. +Integration tests use temporary persistent stores for distributed, restart, +WAL, replication, reconciliation, query, import, and backup regressions. Unit +tests may use in-memory stores. `cargo test --all-targets` can execute Criterion +benchmark binaries and is intentionally not the default correctness command. +Process-kill recovery tests do not simulate every hardware or filesystem failure. ## Container Deployment -```bash -# Build image -podman build -t unirust -f Containerfile . +[Containerfile](Containerfile) builds the binaries with native dependencies and +runs them as a non-root user. [compose.yaml](compose.yaml) supplies a local +three-shard plaintext cluster with the load-test ontology and separate named +data/checkpoint volumes. Those volumes can reside on the same host disk; +production still needs independent storage and authenticated transport. -# Run a single shard -podman run --rm -p 50061:50061 -v unirust-data:/data unirust shard --shard-id 0 +With Podman and a compatible Compose provider installed, from the repository +root: -# Run router -podman run --rm -p 50060:50060 unirust router --shards host.containers.internal:50061 +```bash +podman-compose up --build -d +podman-compose ps +podman-compose logs -f router ``` -### Cluster with Compose +After the router's semantic healthcheck passes, run the optional client workload: -Deploy a 3-shard cluster: ```bash -# Start cluster -podman-compose up -d - -# Check status -podman-compose ps +podman-compose run --rm loadtest +``` -# View router logs -podman-compose logs -f router +The Compose load-test service supplies its own router address and workload +arguments. It uses the `tools` profile, and explicit `run loadtest` selects that +service. The router port is published only on host loopback. Compose enables +hourly checkpoint attempts by default; set `UNIRUST_CHECKPOINT_INTERVAL_SECS` +before startup to select another interval or `0` to disable. This is a Compose +substitution variable, distinct from the binary's +`UNIRUST_ROUTER_CHECKPOINT_INTERVAL_SECS`. Automatic attempts have the retry +consistency limitation described above and do not export backups off-host. -# Run loadtest -podman-compose run --rm loadtest +Stop while preserving volumes: -# Stop and clean up +```bash podman-compose down +``` -# Explicitly delete all persistent shard volumes +To deliberately delete **both shard data and checkpoint volumes**: + +```bash podman-compose down -v ``` diff --git a/compose.yaml b/compose.yaml index 745f7a7..fa2ba80 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,17 +1,20 @@ # Unirust Cluster Compose Configuration # # Deploy a 3-shard cluster with router: +# This configuration uses plaintext internal traffic and no replicas. Separate +# named data/checkpoint volumes do not establish independent failure domains. # # podman-compose up -d # podman-compose logs -f router # # Run loadtest: -# podman-compose run --rm loadtest --router http://router:50060 --count 100000 +# podman-compose run --rm loadtest loadtest --router http://router:50060 --count 100000 --headless +# An explicit run command must retain the first "loadtest" entrypoint argument. # # Stop cluster while preserving data: # podman-compose down # -# Explicitly delete all shard data: +# Explicitly delete shard data AND checkpoint volumes: # podman-compose down -v services: diff --git a/docs/critical-audit-2026-09-05.md b/docs/critical-audit-2026-09-05.md index 07cc978..44d8034 100644 --- a/docs/critical-audit-2026-09-05.md +++ b/docs/critical-audit-2026-09-05.md @@ -1,3 +1,5 @@ +# September 2026 correctness and performance audit + Audit and repairs based on merged commit `3d99fe0d27d7344b08b3bd5b23bb05c75479f1bd`, September 5, 2026. Three parallel reviews covered entity resolution, persistence/recovery, and distributed correctness. All eight original defects were reproduced with persistent storage and have enabled regression coverage. Every accepted record still completes entity resolution before its record data commits. @@ -46,7 +48,7 @@ The comparative ingest run used five persistent shards with the high-throughput Ingest throughput was effectively unchanged in this single comparison; the small throughput and latency differences do not establish a statistically significant change. This workload measures durable ingest, not a universal cross-shard query or reconciliation latency. The selective-query improvement is measured separately above. -CI now passes the Rust 1.98 lints that failed on main (`chunks_exact_to_as_chunks` and `manual_slice_fill`). PR validation explicitly installs native build dependencies and verifies the release package. Release publishing depends on the reusable complete CI workflow, including MSRV, tests, lint, manifests, and production image, and rejects tags that do not match the package version. Workflow syntax is checked with actionlint. +CI now passes the Rust 1.98 lints that failed on main (`chunks_exact_to_as_chunks` and `manual_slice_fill`). PR validation explicitly installs native build dependencies and verifies the release package. Release publishing depends on the reusable complete CI workflow, including MSRV, tests, lint, manifests, and production image, and rejects tags that do not match the package version. Workflow syntax was checked locally with actionlint; the CI manifest step checks shell syntax and the Compose configuration. Validation commands: @@ -59,4 +61,26 @@ cargo package --locked cargo test --release --test audit_performance -- --ignored --nocapture ``` -The correctness suites run by default. Only the manual timing diagnostic is ignored; it has no machine-dependent timing assertion. Distributed benchmarks now include persistent shard ingestion rather than treating the in-memory partition path as evidence of production throughput. +The correctness suites run by default. Of the audit regression tests, only the manual timing diagnostic is ignored; it has no machine-dependent timing assertion. Distributed benchmarks now include persistent shard ingestion rather than treating the in-memory partition path as evidence of production throughput. + + +## Release and measurement provenance + +The repairs shipped in v0.2.0 at commit +`5899e3984e73d28f4dc28e0cd84282cfa0501ffa`. The tables above record local +single-run audit diagnostics, not a controlled hardware benchmark or a claim +that every measurement used the final immutable release artifact. The ingest +comparison preceded the final query/recovery refinements. A final runtime-head +query rerun measured first/warm/after-insert times of 0.057/0.006/0.018 ms at +5,000 records, 0.028/0.006/0.009 ms at 20,000 records and 0.027/0.005/0.008 ms +at 80,000 records. These samples support the removal of the selective query's +full-record scan; they do not establish microsecond latency guarantees. + +Both benchmark clusters used local persistent storage. Synchronous WAL flushing +was enabled, but neither the benchmark nor process-crash regressions constitute +a physical power-cut test. Stable-storage guarantees depend on the operating +system and storage honoring synchronization requests. + +The release passed CI and package publication. The subsequent CI-only commit +`682477f` authenticated protoc downloads after a GitHub API rate limit during +release validation; it is not part of the v0.2.0 crate artifact. diff --git a/examples/cluster.rs b/examples/cluster.rs index a1fd593..bb2c282 100644 --- a/examples/cluster.rs +++ b/examples/cluster.rs @@ -20,32 +20,25 @@ //! //! ## Prerequisites //! -//! Start the cluster first: -//! -//! ```bash -//! # Option 1: Use the cluster script -//! SHARDS=3 ./scripts/cluster.sh start -//! -//! # Option 2: Start manually -//! ./target/release/unirust_shard --listen 127.0.0.1:50061 --shard-id 0 --data-dir /tmp/shard0 --backup-dir /tmp/shard0-backup -//! ./target/release/unirust_shard --listen 127.0.0.1:50062 --shard-id 1 --data-dir /tmp/shard1 --backup-dir /tmp/shard1-backup -//! ./target/release/unirust_shard --listen 127.0.0.1:50063 --shard-id 2 --data-dir /tmp/shard2 --backup-dir /tmp/shard2-backup -//! ./target/release/unirust_router --listen 127.0.0.1:50060 \ -//! --shards 127.0.0.1:50061,127.0.0.1:50062,127.0.0.1:50063 -//! ``` +//! Use a dedicated, fresh persistent cluster: this example replaces its ontology +//! and ingests sample records. Follow the local demonstration in README.md to +//! start three shards and a router. Do not run against an existing dataset. //! //! ## Run It //! //! ```bash -//! cargo run --example cluster +//! cargo run --locked --example cluster -- http://127.0.0.1:50060 //! ``` +//! +//! The router URI is optional and defaults to the address above. The integer +//! interval endpoints below are an application convention, not parsed dates. use unirust_rs::distributed::proto::query_entities_response::Outcome; use unirust_rs::distributed::proto::router_service_client::RouterServiceClient; use unirust_rs::distributed::proto::{ ApplyOntologyRequest, ConstraintConfig, ConstraintKind, IdentityKeyConfig, - IngestRecordsRequest, OntologyConfig, QueryDescriptor, QueryEntitiesRequest, RecordDescriptor, - RecordIdentity, RecordInput, StatsRequest, + IngestRecordsRequest, OntologyConfig, QueryDescriptor, QueryEntitiesRequest, ReconcileRequest, + RecordDescriptor, RecordIdentity, RecordInput, StatsRequest, }; #[tokio::main] @@ -58,10 +51,12 @@ async fn main() -> anyhow::Result<()> { // Step 1: Connect to the Router // ========================================================================= - let router_addr = "http://127.0.0.1:50060"; + let router_addr = std::env::args() + .nth(1) + .unwrap_or_else(|| "http://127.0.0.1:50060".to_string()); println!("Connecting to router at {}...", router_addr); - let mut client = match RouterServiceClient::connect(router_addr).await { + let mut client = match RouterServiceClient::connect(router_addr.clone()).await { Ok(c) => { println!(" Connected successfully!\n"); c @@ -69,20 +64,7 @@ async fn main() -> anyhow::Result<()> { Err(e) => { eprintln!("\nError: Could not connect to router at {}", router_addr); eprintln!(" {}\n", e); - eprintln!("Please start the cluster first:"); - eprintln!(" SHARDS=3 ./scripts/cluster.sh start\n"); - eprintln!("Or manually:"); - eprintln!( - " ./target/release/unirust_shard --listen 127.0.0.1:50061 --shard-id 0 --data-dir /tmp/shard0 --backup-dir /tmp/shard0-backup" - ); - eprintln!( - " ./target/release/unirust_shard --listen 127.0.0.1:50062 --shard-id 1 --data-dir /tmp/shard1 --backup-dir /tmp/shard1-backup" - ); - eprintln!( - " ./target/release/unirust_shard --listen 127.0.0.1:50063 --shard-id 2 --data-dir /tmp/shard2 --backup-dir /tmp/shard2-backup" - ); - eprintln!(" ./target/release/unirust_router --listen 127.0.0.1:50060 \\"); - eprintln!(" --shards 127.0.0.1:50061,127.0.0.1:50062,127.0.0.1:50063"); + eprintln!("Start a fresh persistent demo cluster using the README instructions."); return Err(e.into()); } }; @@ -128,8 +110,8 @@ async fn main() -> anyhow::Result<()> { // ========================================================================= // // Records are automatically distributed across shards based on their - // identity key signatures. Records that might match end up on the same - // shard for efficient local resolution. + // identity key signatures. Placement favors local matches, while explicit + // reconciliation below resolves identities that span multiple shards. println!("Creating sample records..."); @@ -197,7 +179,7 @@ async fn main() -> anyhow::Result<()> { }, RecordDescriptor { attr: "phone".to_string(), - value: format!("555-{:04}", i + 1000), // Different phone -> conflict + value: format!("555-{:04}", i + 1000), // Different phone observation start: 202406, end: 202501, }, @@ -251,6 +233,14 @@ async fn main() -> anyhow::Result<()> { // Step 4: Query Entities // ========================================================================= + // Ingestion resolves locally. Reconcile before expecting cross-shard + // fragments to appear as one canonical entity in the router query. + client + .reconcile(ReconcileRequest { + shard_metadata: Vec::new(), + }) + .await?; + println!("Querying for Person 0..."); let query_response = client @@ -309,7 +299,10 @@ async fn main() -> anyhow::Result<()> { let stats_response = client.get_stats(StatsRequest {}).await?.into_inner(); println!(" - Total records: {}", stats_response.record_count); - println!(" - Total clusters: {}", stats_response.cluster_count); + println!( + " - Sum of shard-local clusters: {}", + stats_response.cluster_count + ); println!(" - Conflicts: {}", stats_response.conflict_count); println!(" - Graph nodes: {}", stats_response.graph_node_count); println!(" - Graph edges: {}", stats_response.graph_edge_count); @@ -323,10 +316,8 @@ async fn main() -> anyhow::Result<()> { ); println!("\n✓ Example completed successfully!"); - println!("\nNext steps:"); - println!(" - Scale up: SHARDS=5 ./scripts/cluster.sh start"); - println!(" - Run loadtest: ./target/release/unirust_loadtest --count 1000000"); - println!(" - Stop cluster: ./scripts/cluster.sh stop"); + println!("Use the demo shell's cleanup instructions to stop its processes."); + println!("See README.md for benchmarking on a separate fresh dataset."); Ok(()) } diff --git a/examples/unirust.toml b/examples/unirust.toml index 9450707..7d2f735 100644 --- a/examples/unirust.toml +++ b/examples/unirust.toml @@ -1,12 +1,14 @@ # Unirust Configuration Example # # Copy this file to /etc/unirust/config.toml or use -c/--config to load it. -# All settings are optional - defaults are used for missing values. +# Missing fields have defaults, but persistent shards still require data and +# checkpoint paths. Fill in the commented deployment-specific settings below. # Environment variables override file settings (prefix: UNIRUST_) # CLI arguments override environment variables # Performance profile (determines internal tuning parameters) -# Options: balanced, low-latency, high-throughput, bulk-ingest, memory-saver, billion-scale +# Options: balanced, low-latency, high-throughput, bulk-ingest, memory-saver, +# billion-scale, billion-scale-high-performance profile = "high-throughput" # Shard node configuration @@ -26,7 +28,7 @@ id = 0 # tls_key = "/etc/unirust/tls/shard-0.key" # tls_client_ca = "/etc/unirust/tls/clients-ca.crt" # Primary-only synchronous replica settings. Bootstrap both volumes from the -# same committed checkpoint before enabling these options. +# same committed checkpoint, or start both empty, before enabling these options. # replica = "https://shard-0-replica:50061" # replication_token_file = "/etc/unirust/replication/shard-0.token" # replica_tls_ca = "/etc/unirust/tls/replicas-ca.crt" @@ -37,7 +39,9 @@ id = 0 # Plaintext replication is rejected unless explicitly enabled for isolated # development: # allow_insecure_replication = false -# Run repair on startup (recovers from unclean shutdown) +# Run RocksDB repair before opening. This is explicit corruption salvage, not +# normal crash recovery; normal WAL replay is automatic. Retain an untouched +# copy of the damaged database before attempting repair. repair = false # Router node configuration @@ -69,18 +73,14 @@ shard_tcp_keepalive_secs = 30 # shard_tls_cert = "/etc/unirust/tls/router-client.crt" # shard_tls_key = "/etc/unirust/tls/router-client.key" -# Advanced storage tuning (RocksDB) -# These settings control the underlying storage engine performance. -# Most users should leave these at defaults. -[storage] -# Block cache size in MB (larger = better read performance) -block_cache_mb = 512 -# Write buffer size in MB (larger = better write batching) -write_buffer_mb = 128 -# Rate limit in MB/s (0 = unlimited, useful for shared systems) -rate_limit_mbps = 0 -# Background compaction threads -max_background_jobs = 4 +# RocksDB tuning is read directly from environment variables by PersistentStore. +# The parsed [storage] configuration is currently not wired to shard startup. +# Examples (set in the process environment, not as TOML fields): +# UNIRUST_BLOCK_CACHE_MB=512 +# UNIRUST_WRITE_BUFFER_MB=128 +# UNIRUST_RATE_LIMIT_MBPS=20 +# UNIRUST_COMPACTION_THREADS=1 +# UNIRUST_FLUSH_THREADS=2 # Advanced reconciliation tuning # Controls cross-shard cluster reconciliation in distributed mode. diff --git a/proto/unirust.proto b/proto/unirust.proto index 619df5b..99b0849 100644 --- a/proto/unirust.proto +++ b/proto/unirust.proto @@ -13,14 +13,14 @@ package unirust; message RecordIdentity { string entity_type = 1; // e.g., "person", "company" string perspective = 2; // Source system, e.g., "crm", "erp" - string uid = 3; // Unique ID within perspective + string uid = 3; // Source UID within this entity_type and perspective } message RecordDescriptor { string attr = 1; // Attribute name, e.g., "email", "phone" string value = 2; // Attribute value - int64 start = 3; // Validity start (epoch or YYYYMM) - int64 end = 4; // Validity end (exclusive) + int64 start = 3; // Inclusive validity start in caller-consistent integer time units + int64 end = 4; // Exclusive validity end; start must be less than end } message RecordInput { @@ -40,8 +40,8 @@ message IngestAssignment { uint32 index = 1; // Correlates to RecordInput.index uint32 shard_id = 2; // Shard that owns this record uint32 record_id = 3; // Assigned record ID - uint32 cluster_id = 4; // Assigned cluster ID - string cluster_key = 5; // Human-readable cluster key + uint32 cluster_id = 4; // Local ingest assignment, distinct from a canonical query ID + string cluster_key = 5; // Currently empty on ingest; display keys are derived on query } message IngestRecordsResponse { @@ -69,6 +69,9 @@ message GoldenDescriptor { } message QueryMatch { + // Ordinary matches and multi-shard router conflicts use a canonical global + // shard and local record anchor. The legacy direct-shard conflict response + // uses local cluster IDs; a single-shard router forwards that response. uint32 shard_id = 1; uint32 cluster_id = 2; int64 start = 3; @@ -145,7 +148,7 @@ message RecordRef { } message ConflictSummary { - string kind = 1; // "direct" or "indirect" + string kind = 1; // e.g. "direct", "indirect", "indirect_cross_shard" string attribute = 2; // Conflicting attribute int64 start = 3; int64 end = 4; @@ -244,13 +247,14 @@ service RouterService { // --- Advanced Operations --- - // Ingest from URL (bulk loading) + // Deprecated compatibility endpoint; currently returns UNIMPLEMENTED. rpc IngestRecordsFromUrl(IngestRecordsFromUrlRequest) returns (IngestRecordsResponse); // Get config version rpc GetConfigVersion(ConfigVersionRequest) returns (ConfigVersionResponse); - // Create checkpoint (for backup/restore) + // Prepare/finalize a checkpoint generation for backup/restore. Failed retries + // may reuse older shard snapshots; see CheckpointResponse.committed. rpc Checkpoint(CheckpointRequest) returns (CheckpointResponse); // Get detailed metrics @@ -313,7 +317,7 @@ message ConfigVersionResponse { string restore_generation = 7; uint32 restore_shard_count = 8; ShardRole shard_role = 9; - // SHA-256 over every logical RocksDB column-family key/value pair. + // SHA-256 over the configured Unirust durable-state column-family key/value pairs. bytes durable_state_digest = 10; uint32 query_fragment_protocol_version = 11; } @@ -350,6 +354,8 @@ message MarkSourceReservationsBackfilledResponse {} // --- Checkpoint --- message CheckpointRequest { + // Generation path relative to each shard's checkpoint root. The router chooses + // a generation when empty; this is not an arbitrary absolute filesystem path. string path = 1; // Internal router-to-shard coordination fields. External clients call the // router with these left at their zero values. @@ -361,6 +367,9 @@ message CheckpointRequest { message CheckpointResponse { repeated string paths = 1; string generation = 2; + // Confirms finalization, not a common capture time after interrupted retries. + // Existing prepared shard snapshots are reused for the same generation; the + // router's mutation gate is released between failed attempts. bool committed = 3; } @@ -461,9 +470,9 @@ message RouterImportRecordsRequest { // --- Cross-Shard Reconciliation --- message GlobalClusterId { - uint32 shard_id = 1; - uint32 local_id = 2; - uint32 version = 3; + uint32 shard_id = 1; // Validated to fit 16 bits + uint32 local_id = 2; // Durable local record anchor for linker-created IDs + uint32 version = 3; // Validated to fit 16 bits; current linker-created IDs use zero } message IdentityKeySignature { @@ -485,8 +494,8 @@ message ClusterBoundaryEntry { uint32 shard_id = 4; // Strong ID hashes per perspective for cross-shard conflict detection. // Key: perspective name hash, Value: hash of strong ID values. - // Two clusters conflict if they share a perspective but have different values. - // Deprecated: retained for rolling compatibility with pre-0.2 nodes. + // Legacy fallback lacks the exact temporal observations available below. + // Retained in the schema; live protocol checks reject incompatible nodes. map perspective_strong_ids = 5; // Exact temporal observations used for authoritative merge guards. repeated BoundaryStrongId strong_ids = 6; @@ -541,6 +550,8 @@ message ClearDirtyKeysResponse { } message ReconcileRequest { + // External callers leave this empty. The router rejects supplied metadata and + // fetches authoritative boundaries directly from shards. repeated BoundaryMetadata shard_metadata = 1; } @@ -624,6 +635,7 @@ service ShardService { rpc MarkSourceReservationsBackfilled(MarkSourceReservationsBackfilledRequest) returns (MarkSourceReservationsBackfilledResponse); rpc IngestRecords(IngestRecordsRequest) returns (IngestRecordsResponse); rpc IngestRecordsStream(stream IngestRecordsChunk) returns (IngestRecordsResponse); + // Deprecated compatibility endpoint; currently returns UNIMPLEMENTED. rpc IngestRecordsFromUrl(IngestRecordsFromUrlRequest) returns (IngestRecordsResponse); rpc QueryEntities(QueryEntitiesRequest) returns (QueryEntitiesResponse); rpc QueryEntityFragments(QueryEntityFragmentsRequest) returns (QueryEntityFragmentsResponse); diff --git a/scripts/cluster.sh b/scripts/cluster.sh index 58af550..9f0b61a 100755 --- a/scripts/cluster.sh +++ b/scripts/cluster.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash +# Local persistent cluster helper. BACKUP_DIR is a separate path, which may still +# share the data directory's physical disk. Reset removes DATA_DIR, not BACKUP_DIR. +# Automatic checkpoint retries reuse prepared snapshots; quiesce writers across +# the complete attempt/retry sequence when a common capture point is required. set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -38,7 +42,7 @@ Environment: Default: $ROOT_DIR/examples/loadtest-ontology.json CONFIG_VERSION=optional-version-string CHECKPOINT_INTERVAL_SECS=0 Automatic coordinated checkpoint interval (0 disables) - PROFILE=balanced|low-latency|high-throughput|bulk-ingest|memory-saver|billion-scale + PROFILE=balanced|low-latency|high-throughput|bulk-ingest|memory-saver|billion-scale|billion-scale-high-performance REPAIR=0|1 CARGO_FEATURES=comma-separated-cargo-features SHARD_WAIT_SECS=10 diff --git a/scripts/podman_cluster.sh b/scripts/podman_cluster.sh index 0d8e522..dcb8cb5 100755 --- a/scripts/podman_cluster.sh +++ b/scripts/podman_cluster.sh @@ -1,4 +1,10 @@ #!/usr/bin/env bash +# Local Podman cluster helper. The router's -p mapping uses Podman's default host +# bind behavior; the "localhost" status text does not restrict published access. +# Separate named backup volumes can share the data volumes' host/disk. Reset +# deletes both sets. No TLS or passive replicas are configured by this helper. +# Checkpoint retries reuse prepared snapshots; keep writers quiesced across the +# attempt/retry sequence when a common capture point is required. set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" diff --git a/src/bin/unirust_loadtest.rs b/src/bin/unirust_loadtest.rs index a21d51e..0484468 100644 --- a/src/bin/unirust_loadtest.rs +++ b/src/bin/unirust_loadtest.rs @@ -1648,18 +1648,28 @@ impl LoadTestMetrics { throughput / config.stream_count as f64 )); - // Bottleneck analysis - report.push_str("\n## Bottleneck Analysis\n"); + // Diagnostic prompts; these thresholds do not identify a root cause. + report.push_str("\n## Diagnostic Prompts\n"); if throughput < 1000.0 { - report.push_str(" ⚠ Throughput < 1000 rec/sec - likely I/O bound\n"); - report.push_str(" - Check RocksDB sync writes\n"); + report.push_str( + " ⚠ Throughput < 1000 rec/sec - investigate CPU, storage and RPC costs\n", + ); + report.push_str( + " - Measure RocksDB sync latency; preserve durable acknowledgements\n", + ); report.push_str(" - Consider increasing batch size\n"); report.push_str(" - Check disk I/O with iostat\n"); } if batch_latency_avg > 500.0 { - report.push_str(" ⚠ RPC latency > 500ms - processing bottleneck\n"); - report.push_str(" - Check conflict detection overhead\n"); - report.push_str(" - Consider profiling with UNIRUST_PROFILE=1\n"); + report.push_str( + " ⚠ Average RPC latency > 500ms - inspect time spent across the request path\n", + ); + report.push_str( + " - Check routing, entity resolution, reconciliation and storage costs\n", + ); + report.push_str( + " - Use a CPU/I/O profiler; UNIRUST_PROFILE selects a named tuning preset\n", + ); } let max_stream_latency = self .stream_stats @@ -1675,7 +1685,9 @@ impl LoadTestMetrics { .unwrap_or(0); if max_stream_latency > min_stream_latency * 2 && min_stream_latency > 0 { report.push_str(" ⚠ Stream latency imbalance detected\n"); - report.push_str(" - Uneven shard distribution or hot keys\n"); + report.push_str( + " - Check for uneven shard distribution, hot keys or scheduling delays\n", + ); } report.push_str( diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 4548bcc..4267b04 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -134,7 +134,7 @@ pub const DEFAULT_WAL_COALESCE_RECORDS: usize = 5000; pub const DEFAULT_WAL_CHANNEL_CAPACITY: usize = 5000; // ============================================================================= -// Linker State Cache Defaults (for billion-scale) +// Reserved Linker State Cache Defaults (limits are not currently enforced) // ============================================================================= /// Default cluster IDs cache capacity diff --git a/src/config/mod.rs b/src/config/mod.rs index cf25426..a56852c 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -303,9 +303,10 @@ pub enum Profile { BulkIngest, /// Reduced memory footprint MemorySaver, - /// For billion-scale datasets with persistent storage + /// Request persistent DSU and tiered index caches when supported by the store. BillionScale, - /// For billion-scale datasets with larger caches (production default) + /// Larger persistent-backend caches and candidate budgets (node config default). + /// Correctness-critical linker state still grows in memory. #[default] BillionScaleHighPerformance, } diff --git a/src/config/tuning.rs b/src/config/tuning.rs index 1daf6d3..fdd0b00 100644 --- a/src/config/tuning.rs +++ b/src/config/tuning.rs @@ -1,7 +1,9 @@ //! Streaming engine tuning configuration. //! //! These are internal tuning parameters for the entity resolution engine. -//! Most users should select a `Profile` instead of tuning these directly. +//! Select a [`TuningProfile`] for an embedded engine or [`super::Profile`] in node +//! configuration before adjusting individual parameters. Presets are workload +//! tradeoffs, not throughput or total-memory guarantees. use crate::conflicts::ConflictAlgorithm; use crate::dsu::PersistentDSUConfig; @@ -19,10 +21,9 @@ pub struct StreamingTuning { pub adaptive_mid_cap: usize, pub deferred_reconciliation: bool, pub hot_key_threshold: usize, - /// Enable stochastic candidate sampling (SPER-inspired optimization). - /// When candidates exceed sampling_threshold, sample with probability - /// proportional to temporal overlap instead of hard cutoff. - /// This maintains expected match quality while reducing computation. + /// Enable deterministic, hash-based candidate sampling above `sampling_threshold`. + /// Selection is weighted by temporal overlap. Sampling can omit matching + /// candidates and affect resolution; it is not equivalent to exhaustive linking. pub stochastic_sampling: bool, /// Threshold above which stochastic sampling kicks in. pub sampling_threshold: usize, @@ -30,19 +31,19 @@ pub struct StreamingTuning { pub sampling_target: usize, /// Configuration for persistent DSU (used when `use_persistent_dsu` is true) pub dsu_config: Option, - /// Whether to use persistent DSU for billion-scale datasets + /// Request a RocksDB-backed DSU when the store exposes a shared database. pub use_persistent_dsu: bool, /// Configuration for tiered index storage (used when `use_tiered_index` is true) pub tier_config: Option, - /// Whether to use tiered index for billion-scale datasets + /// Request hot/warm index caches with RocksDB-backed cold buckets when available. pub use_tiered_index: bool, /// Shard ID for this node (used for boundary tracking in distributed mode) pub shard_id: u16, /// Whether to track boundary signatures for cross-shard reconciliation. /// Default false - enable only in distributed mode to reduce memory overhead. pub enable_boundary_tracking: bool, - /// Configuration for linker state memory management (LRU caches). - /// Default: None (uses HashMap, unlimited capacity). + /// Reserved linker-state cache configuration. Limits are currently not enforced; + /// supplying them emits a warning and retains correctness-critical state in memory. pub linker_state_config: Option, } @@ -50,14 +51,14 @@ pub struct StreamingTuning { #[derive(Debug, Clone)] pub struct ConflictTuning { /// The algorithm to use for conflict detection. - /// - `SweepLine`: O(n log n), best for diverse time boundaries - /// - `AtomicIntervals`: O(atoms × n), best for overlapping intervals - /// - `Auto`: Automatically select based on overlap ratio heuristic + /// - `SweepLine`: Sweep sorted temporal boundary events + /// - `AtomicIntervals`: Evaluate observations in atomic intervals + /// - `Auto`: Select using the fraction of distinct temporal boundaries pub algorithm: ConflictAlgorithmChoice, /// Threshold for auto-selection: if the ratio of unique boundaries - /// to total descriptors is below this, use AtomicIntervals. - /// Default: 0.5 (if < 50% unique boundaries, use atomic intervals) + /// to twice the descriptor count is below this, use AtomicIntervals. + /// Default: 0.5. Shared boundaries are a heuristic, not a runtime guarantee. pub auto_overlap_threshold: f64, } @@ -83,7 +84,7 @@ impl Default for ConflictTuning { } impl ConflictTuning { - /// Create tuning optimized for high-overlap workloads (production default) + /// Select atomic intervals for workloads with many shared temporal boundaries. pub fn high_overlap() -> Self { Self { algorithm: ConflictAlgorithmChoice::AtomicIntervals, @@ -116,17 +117,17 @@ impl ConflictTuning { return ConflictAlgorithm::SweepLine; } - // Ratio of unique boundaries to total descriptors + // Ratio of unique boundaries to all descriptor endpoints // Each descriptor contributes 2 boundaries (start, end) // If many share the same boundaries, the ratio is low let max_boundaries = total_descriptors * 2; let ratio = unique_boundaries as f64 / max_boundaries as f64; if ratio < self.auto_overlap_threshold { - // High overlap detected - atomic intervals is faster + // Prefer atomic intervals when many boundaries are shared. ConflictAlgorithm::AtomicIntervals } else { - // Low overlap - sweep line is faster + // Prefer the sweep line when boundaries are more diverse. ConflictAlgorithm::SweepLine } } @@ -142,9 +143,9 @@ pub enum TuningProfile { HighThroughput, BulkIngest, MemorySaver, - /// Optimized for billion-scale datasets with persistent DSU + /// Request persistent DSU and tiered index caches when the store supports them. BillionScale, - /// High-performance billion-scale with larger caches + /// Request persistent backends with larger caches and candidate budgets. BillionScaleHighPerformance, } @@ -282,7 +283,8 @@ impl StreamingTuning { } } - /// Configuration optimized for billion-scale datasets with persistent DSU and tiered index + /// Request persistent DSU and tiered index backends with default cache sizes. + /// Correctness-critical linker maps remain in memory and grow with the dataset. pub fn billion_scale() -> Self { Self { candidate_cap: DEFAULT_CANDIDATE_CAP, @@ -309,7 +311,8 @@ impl StreamingTuning { } } - /// High-performance configuration for billion-scale with larger caches + /// Request persistent backends with larger caches and candidate budgets. + /// The name does not imply a supported dataset size or total-memory bound. pub fn billion_scale_high_performance() -> Self { Self { candidate_cap: 4000, @@ -354,15 +357,15 @@ impl StreamingTuning { /// retains unbounded state and emits a warning. #[derive(Debug, Clone)] pub struct LinkerStateConfig { - /// Maximum number of cluster ID mappings to keep in memory. + /// Requested cluster-ID mapping capacity; currently not enforced. pub cluster_ids_capacity: usize, - /// Maximum number of global cluster ID mappings to keep in memory. + /// Requested global-ID mapping capacity; currently not enforced. pub global_ids_capacity: usize, - /// Maximum number of strong ID summaries to keep in memory. + /// Requested strong-ID summary capacity; currently not enforced. pub summaries_capacity: usize, - /// Maximum number of record perspectives to keep in memory. + /// Requested record-perspective capacity; currently not enforced. pub perspectives_capacity: usize, - /// Size of dirty buffer before flushing to disk (if persistence enabled). + /// Reserved linker-state dirty-buffer threshold; currently unused. pub dirty_buffer_size: usize, } @@ -379,8 +382,7 @@ impl Default for LinkerStateConfig { } impl LinkerStateConfig { - /// Configuration optimized for memory-constrained environments. - /// Uses smaller caches (~200MB total). + /// Smaller requested capacities; current linker implementations do not enforce them. pub fn memory_saver() -> Self { Self { cluster_ids_capacity: 500_000, @@ -391,8 +393,7 @@ impl LinkerStateConfig { } } - /// Configuration optimized for high-performance environments. - /// Uses larger caches (~1GB total). + /// Larger requested capacities; current linker implementations do not enforce them. pub fn high_performance() -> Self { Self { cluster_ids_capacity: 20_000_000, @@ -403,8 +404,8 @@ impl LinkerStateConfig { } } - /// Unlimited capacity (disables LRU eviction). - /// Only use for small datasets where everything fits in memory. + /// Set requested capacities to `usize::MAX`. Linker state is currently retained + /// regardless of these values because durable spill is not implemented. pub fn unlimited() -> Self { Self { cluster_ids_capacity: usize::MAX, diff --git a/src/distributed.rs b/src/distributed.rs index abe407d..bb84952 100644 --- a/src/distributed.rs +++ b/src/distributed.rs @@ -1051,14 +1051,13 @@ fn boundary_index_from_metadata( #[derive(Clone)] pub struct ShardNode { shard_id: u32, - /// Uses parking_lot RwLock (faster for short critical sections than tokio's async RwLock) + /// Reader/writer lock protecting the main shard engine. unirust: Arc>, - /// Partitioned Unirust for high-throughput parallel processing (optional) - /// Uses per-partition locks for TRUE parallel processing - no global lock! - /// Wrapped in RwLock to allow rebuilding when ontology changes - /// Inner Arc allows cloning for use across await points + /// Optional in-memory partition processor; persistent shards use the main engine. + /// A publication lock permits ontology replacement, while partition mutexes + /// serialize work within each partition. The inner Arc can span await points. partitioned: Arc>>>, - /// Concurrent interner for lock-free record building + /// Interner with sharded locking for concurrent in-memory record building. concurrent_interner: Arc, tuning: StreamingTuning, ontology_config: Arc>, @@ -1095,8 +1094,8 @@ const RECONCILIATION_KEY_CHUNK: usize = 10_000; const MERGE_APPLICATION_CHUNK: usize = 50_000; const CONFLICT_APPLICATION_CHUNK: usize = 10_000; -/// Check if partitioned processing is enabled (default: true) -/// Set UNIRUST_PARTITIONED=0 to disable +/// Check whether in-memory partition processing is requested (default: true). +/// Set UNIRUST_PARTITIONED=0 to disable. Persistent shards do not use this path. fn is_partitioned_enabled() -> bool { std::env::var("UNIRUST_PARTITIONED") .map(|v| v != "0" && v.to_lowercase() != "false") @@ -1414,8 +1413,7 @@ impl ShardNode { let recovered_cross_shard_conflicts = unirust.load_cross_shard_conflicts()?; let unirust = Arc::new(parking_lot::RwLock::new(unirust)); - // Create partitioned processor if enabled - no RwLock needed! - // ParallelPartitionedUnirust has per-partition Mutexes for true parallelism + // The optional in-memory processor locks each partition during its batch. let partitioned = if use_partitioned { let partition_config = PartitionConfig::for_cores(num_partitions); let partitioned_unirust = ParallelPartitionedUnirust::new_with_interner( @@ -1481,8 +1479,7 @@ impl ShardNode { tuning.clone(), ))); - // Create partitioned processor if enabled - no RwLock needed! - // ParallelPartitionedUnirust has per-partition Mutexes for true parallelism + // The optional in-memory processor locks each partition during its batch. let partitioned = if use_partitioned { let partition_config = PartitionConfig::for_cores(num_partitions); let partitioned_unirust = ParallelPartitionedUnirust::new_with_interner( @@ -2226,7 +2223,7 @@ fn ingest_worker_index(record: &proto::RecordInput, worker_count: usize) -> usiz (hasher.finish() as usize) % worker_count } -/// Spawn ingest workers using parking_lot::RwLock for faster locking. +/// Spawn ingest workers that acquire the main engine's write lock for each batch. fn spawn_ingest_workers( unirust: Arc>, shard_id: u32, @@ -2291,7 +2288,8 @@ async fn dispatch_ingest_records( } /// Compute partition ID for a record using the interner and ontology's identity keys. -/// This ensures records with the same identity key values end up in the same partition. +/// Hashes values for the first identity key in descriptor order, or falls back to UID. +/// This routing heuristic does not guarantee colocation for every matching key. fn compute_partition_id_for_record( record: &Record, ontology: &crate::Ontology, @@ -2338,13 +2336,9 @@ fn compute_partition_id_for_record( (hasher.finish() as usize) % partition_count } -/// Dispatch records using the partitioned architecture for maximum throughput. -/// This bypasses the worker queue entirely and processes partitions in parallel. -/// -/// Performance architecture: -/// High-performance dispatch with REAL entity resolution. -/// Uses parallel partitioned processing for maximum throughput while -/// maintaining full entity resolution correctness. +/// Dispatch in-memory records directly to partition processing, bypassing the main +/// engine's worker queue. Partitions run concurrently with per-partition locks, +/// and every record still goes through the partition linker's entity resolution. async fn dispatch_ingest_partitioned( partitioned: &Arc, interner: &Arc, @@ -2430,8 +2424,7 @@ fn process_ingest_batch( indices.push(record.index); } - // Fast path: stream_records skips graph updates and conflict detection - // This is 10x+ faster than stream_records_update_graph + // Resolve every record without materializing graph exports or conflict summaries. let cluster_assignments = unirust .stream_records(record_inputs) .map_err(|err| Status::internal(err.to_string()))?; @@ -6545,7 +6538,7 @@ impl proto::router_service_server::RouterService for RouterNode { } } -/// RouterService implementation for Arc to allow use with gRPC server +/// RouterService implementation for `Arc` to allow use with gRPC server. /// while returning Arc from connect methods for background task spawning. #[tonic::async_trait] impl proto::router_service_server::RouterService for Arc { diff --git a/src/dsu.rs b/src/dsu.rs index fcfae99..a227653 100644 --- a/src/dsu.rs +++ b/src/dsu.rs @@ -5,7 +5,7 @@ //! //! This module provides two implementations: //! - `TemporalDSU`: In-memory implementation for smaller datasets -//! - `PersistentTemporalDSU`: RocksDB-backed implementation for billion-scale datasets +//! - `PersistentTemporalDSU`: RocksDB-backed entries with in-memory caches and write buffers use crate::model::{ClusterId, RecordId}; use crate::temporal::Interval; @@ -164,8 +164,8 @@ impl TemporalDSU { /// Find the root of a record (with path compression via path halving) /// Returns the record itself if not in DSU (treats untracked records as self-roots) /// - /// Uses a root cache for O(1) lookup of recently found roots, dramatically - /// improving performance when the same records are queried repeatedly. + /// A validated root-cache hit avoids walking the parent chain; stale entries + /// fall back to path traversal and compression. #[inline] pub fn find(&mut self, record_id: RecordId) -> RecordId { // Fast path: check root cache first (common in streaming) @@ -522,13 +522,13 @@ impl Default for Clusters { /// Configuration for persistent DSU cache sizes #[derive(Debug, Clone)] pub struct PersistentDSUConfig { - /// Maximum entries in parent cache (default: 5M, ~80MB) + /// Maximum entries in parent cache (default: 5M). pub parent_cache_size: usize, - /// Maximum entries in rank cache (default: 1M, ~12MB) + /// Maximum entries in rank cache (default: 1M). pub rank_cache_size: usize, - /// Maximum entries in guards cache (default: 100K) + /// Maximum entries in guards cache (default: 500K); payload sizes vary. pub guards_cache_size: usize, - /// Size of dirty write buffer before flush (default: 100K) + /// Dirty-entry threshold before flush (default: 200K). pub dirty_buffer_size: usize, /// Enable path compression writes to disk (default: false for cold paths) pub persist_path_compression: bool, @@ -539,7 +539,7 @@ impl Default for PersistentDSUConfig { Self { parent_cache_size: 5_000_000, rank_cache_size: 1_000_000, - guards_cache_size: 500_000, // Increased: ~50MB for 500K entries + guards_cache_size: 500_000, // Entry count, not a byte limit. dirty_buffer_size: 200_000, // Increased: fewer flushes persist_path_compression: false, } @@ -570,16 +570,17 @@ impl PersistentDSUConfig { } } -/// RocksDB-backed Disjoint Set Union for billion-scale entity resolution. +/// RocksDB-backed Disjoint Set Union with cached entries and buffered writes. /// /// Uses LRU caches for hot paths with RocksDB for persistent storage. -/// Memory usage: ~2GB instead of 60-80GB for 1B entities. +/// Cache limits count entries, not total bytes. Memory also depends on buffered +/// writes, guard payloads, RocksDB settings, and the caller's linker state. pub struct PersistentTemporalDSU { /// Reference to the RocksDB database db: Arc, - /// LRU cache for parent lookups (5M entries, ~80MB) + /// LRU cache for parent lookups, sized by the DSU configuration. parent_cache: LruCache, - /// LRU cache for rank lookups (1M entries, ~12MB) + /// LRU cache for rank lookups, sized by the DSU configuration. rank_cache: LruCache, /// LRU cache for guards guards_cache: LruCache>, @@ -1239,7 +1240,7 @@ pub struct PersistentDSUStats { pub enum DsuBackend { /// In-memory DSU for smaller datasets (< 100M entities) InMemory(TemporalDSU), - /// Persistent DSU for billion-scale datasets (boxed to reduce enum size) + /// RocksDB-backed DSU (boxed to reduce enum size). Persistent(Box), } diff --git a/src/index.rs b/src/index.rs index 7a89ea9..f4ed7b7 100644 --- a/src/index.rs +++ b/src/index.rs @@ -856,20 +856,22 @@ use std::num::NonZeroUsize; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; -/// Configuration for tiered index storage +/// Configuration for bucket caches. Entry capacities do not bound bucket size, +/// metadata maps, or total engine memory. Without RocksDB, overflow is retained +/// rather than evicted when both resident tiers fill. #[derive(Debug, Clone)] pub struct TierConfig { - /// Maximum entries in hot tier (default: 100K keys, ~2GB) + /// Hot-key count that triggers demotion (default: 100K keys). pub hot_tier_capacity: usize, - /// Maximum entries in warm tier (default: 100K keys, ~2GB) + /// Warm LRU entry capacity (default: 100K keys). pub warm_tier_capacity: usize, - /// Score threshold for hot tier (default: 0.5) + /// Reserved hot-score threshold; current capacity-driven demotion does not use it. pub hot_threshold: f64, - /// Score threshold for warm tier (default: 0.2) + /// Reserved warm-score threshold; current capacity-driven demotion does not use it. pub warm_threshold: f64, - /// Maximum cardinality before forcing cold storage (default: 10K) + /// Cardinality input to the demotion score (default: 10K); not a hard bucket cap. pub max_hot_cardinality: u32, - /// Interval between tier management runs (seconds) + /// Reserved interval; current tier management runs when hot capacity is exceeded. pub tier_management_interval_secs: u64, } @@ -887,7 +889,7 @@ impl Default for TierConfig { } impl TierConfig { - /// Memory-optimized configuration (~500MB total) + /// Smaller hot/warm entry capacities; total bytes depend on bucket contents. pub fn memory_saver() -> Self { Self { hot_tier_capacity: 20_000, @@ -899,7 +901,7 @@ impl TierConfig { } } - /// High-performance configuration (~8GB total) + /// Larger hot/warm entry capacities and a higher cardinality scoring threshold. pub fn high_performance() -> Self { Self { hot_tier_capacity: 500_000, @@ -1018,7 +1020,7 @@ impl CompactBucket { /// Tiered identity key index with hot/warm/cold storage pub struct TieredIdentityKeyIndex { - /// Hot tier: Full KeyBucket with IntervalTree for O(log n) queries + /// Hot tier: full KeyBucket with sorted interval indexes and overlap scans. hot: HashMap, /// Access statistics for hot tier keys hot_stats: HashMap, @@ -1459,7 +1461,7 @@ pub struct TieredIndexStats { pub enum IndexBackend { /// In-memory index for smaller datasets InMemory(IdentityKeyIndex), - /// Tiered index with hot/warm/cold tiers for billion-scale + /// Hot/warm caches with RocksDB-backed cold buckets; metadata remains in memory. Tiered(Box), } diff --git a/src/lib.rs b/src/lib.rs index b8f4a07..7da236f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,25 +1,41 @@ //! # Unirust //! -//! A simple, fast entity resolution engine with temporal awareness. +//! Temporal entity resolution with persistent storage and distributed shard/router services. //! //! ## Quick Start //! -//! ```ignore -//! use unirust::{Unirust, Ontology, Record}; +//! This example embeds a persistent engine in one process. The [`distributed`] module +//! provides the shard/router services used for distributed deployments. //! -//! // Create engine with ontology (matching rules) -//! let ontology = Ontology::new(); -//! let mut engine = Unirust::new(ontology); +//! ```no_run +//! use unirust_rs::{ +//! Descriptor, Interval, Ontology, PersistentStore, QueryDescriptor, Record, +//! RecordId, RecordIdentity, Unirust, +//! }; +//! use unirust_rs::ontology::IdentityKey; //! -//! // Ingest records - returns assignments and detected conflicts -//! let result = engine.ingest(records)?; -//! println!("Assigned {} records to {} clusters", result.assignments.len(), result.cluster_count); +//! # fn main() -> anyhow::Result<()> { +//! let mut ontology = Ontology::new(); +//! ontology.add_identity_key(IdentityKey::from_names(vec!["email"], "email")); +//! let store = PersistentStore::open("unirust-data")?; +//! let mut engine = Unirust::with_store(ontology, store); //! -//! // Query master entities -//! let matches = engine.query(&[QueryDescriptor { attr, value }], interval)?; +//! // Descriptor IDs belong to this store's interner. +//! let attr = engine.intern_attr("email"); +//! let value = engine.intern_value("alice@example.com"); +//! let interval = Interval::new(0, 10)?; +//! let record = Record::new( +//! RecordId(0), // Request an automatically allocated record ID. +//! RecordIdentity::new("person".into(), "crm".into(), "alice".into()), +//! vec![Descriptor::new(attr, value, interval)], +//! ); //! -//! // Export knowledge graph -//! let graph = engine.graph()?; +//! let result = engine.ingest(vec![record])?; +//! assert_eq!(result.assignments.len(), 1); +//! let outcome = engine.query(&[QueryDescriptor { attr, value }], interval)?; +//! println!("{outcome:?}"); +//! # Ok(()) +//! # } //! ``` //! //! ## API Design @@ -29,7 +45,7 @@ //! - `query()` - Find master entities by attributes //! - `clusters()` - Get all cluster assignments //! - `graph()` - Export knowledge graph -//! - `checkpoint()` - Persist state to disk +//! - `checkpoint()` - Synchronize the current persistent store //! - `stats()` - Get metrics and statistics pub mod backup; @@ -118,7 +134,8 @@ pub struct Stats { pub record_count: usize, /// Number of clusters pub cluster_count: usize, - /// Records linked per second (recent average) + /// Legacy field containing the current linker's cumulative records-linked count + /// as a float; this is not normalized by elapsed time and is not a rate. pub records_per_second: f64, /// Number of merges performed pub merges_performed: u64, @@ -216,7 +233,8 @@ struct QueryCache { } impl Unirust { - /// Create a new Unirust instance + /// Create an in-memory engine. Use [`Self::with_store`] with [`PersistentStore`] + /// for durable storage. pub fn new(ontology: Ontology) -> Self { Self::with_store(ontology, Store::new()) } @@ -338,17 +356,11 @@ impl Unirust { /// Ingest records and return cluster assignments with detected conflicts. /// /// This is the primary method for adding data to the engine. It: - /// 1. Adds records to the store - /// 2. Links them into clusters based on identity keys - /// 3. Detects conflicts within clusters + /// 1. Stages new records + /// 2. Links them using identity keys and temporal guards + /// 3. Detects conflicts, then commits records and synchronizes the store /// - /// # Example - /// ```ignore - /// let result = engine.ingest(records)?; - /// for assignment in result.assignments { - /// println!("Record {} -> Cluster {}", assignment.record_id.0, assignment.cluster_id.0); - /// } - /// ``` + /// See the crate-level example for persistent ingestion. pub fn ingest(&mut self, records: Vec) -> anyhow::Result { self.ingest_internal(records, false) } @@ -425,8 +437,8 @@ impl Unirust { .iter() .filter_map(|(id, inserted)| inserted.then_some(*id)) .collect(); - // In-memory stores can lend records directly. Persistent stores return - // owned records from their cache for the parallel extraction phase. + // Built-in stores lend staged records directly. Custom stores that + // cannot lend them use the owned-record fallback below. let records_to_link: Vec<&Record> = staged_info .iter() .filter(|(_, inserted)| *inserted) @@ -577,9 +589,11 @@ impl Unirust { Ok(graph) } - /// Persist all state to disk (for persistent stores). + /// Synchronize staged records and the available linker-ID snapshot to the store. /// - /// This flushes the linker state and any staged records. + /// This does not create a backup directory or coordinate a distributed checkpoint. + /// Successful high-level ingestion already synchronizes its committed records. + /// Use [`Self::checkpoint_linker_state`] to also flush the persistent DSU buffers. pub fn checkpoint(&mut self) -> anyhow::Result<()> { // Flush linker state if available if let Some(streaming) = &self.streaming { @@ -594,15 +608,20 @@ impl Unirust { Ok(()) } - /// Create a durable checkpoint at a specific path (advanced). + /// Create a store-level checkpoint of already-written state at a specific path. /// - /// For most cases, use `checkpoint()` instead. + /// This delegates to the storage backend without flushing staged records or + /// linker buffers. Distributed backups require the coordinated cluster checkpoint + /// protocol; a single store checkpoint does not capture the other shards or WALs. #[doc(hidden)] pub fn checkpoint_to_path(&self, path: &std::path::Path) -> anyhow::Result<()> { self.store.checkpoint(path) } - /// Get engine statistics and metrics. + /// Get store size and metrics from the current in-process linker. + /// + /// Linker counters include recovery replay work and reset when the linker is + /// rebuilt. Cluster count and linker counters are zero before initialization. pub fn stats(&self) -> Stats { let mut stats = Stats { record_count: self.store.len(), @@ -620,7 +639,7 @@ impl Unirust { stats.conflicts_detected = snapshot.conflicts_detected; stats.hot_key_exits = snapshot.hot_key_exits; stats.records_per_second = if snapshot.records_linked > 0 { - snapshot.records_linked as f64 // Approximation + snapshot.records_linked as f64 // Legacy field reports a count, not a rate. } else { 0.0 }; @@ -837,9 +856,9 @@ impl Unirust { /// Stream records and return the cluster assignment for each record. /// - /// Uses parallel extraction for batches >= 100 records, providing 20-40% better - /// scaling behavior at large dataset sizes. Falls back to sequential processing - /// for smaller batches where parallelism overhead isn't worth it. + /// Uses parallel key extraction when at least 100 records are newly inserted, + /// followed by sequential linking with temporal guards. Smaller batches use + /// sequential extraction and linking. pub fn stream_records( &mut self, records: Vec, @@ -876,8 +895,8 @@ impl Unirust { .iter() .filter_map(|(id, inserted)| inserted.then_some(*id)) .collect(); - // In-memory stores can lend records directly. Persistent stores return - // owned records from their cache for the parallel extraction phase. + // Built-in stores lend staged records directly. Custom stores that + // cannot lend them use the owned-record fallback below. let records_to_link: Vec<&Record> = staged_info .iter() .filter(|(_, inserted)| *inserted) @@ -1618,8 +1637,8 @@ impl Unirust { } /// Apply a cross-shard cluster merge. - /// Records that records in `secondary` cluster should now be considered part of `primary`. - /// Returns the number of affected records. + /// Record that the `secondary` cluster should now be considered part of `primary`. + /// Returns the number of local global-ID mappings updated, not the member count. pub fn apply_cross_shard_merge( &mut self, primary: GlobalClusterId, @@ -1742,9 +1761,11 @@ impl Unirust { self.streaming.as_ref().map(|s| s.next_cluster_id()) } - /// Flush linker state to persistent storage. - /// This saves cluster_ids, global_cluster_ids, and next_cluster_id. - /// Call this periodically or before shutdown for restart recovery. + /// Write a snapshot of local/global cluster-ID mappings and the next local ID. + /// + /// This does not synchronize the store by itself. Normal recovery rebuilds the + /// linker from committed records; this optional snapshot supports explicit ID-map + /// restoration with [`Self::restore_linker_state`]. /// Returns an error if streaming is not initialized or persistence is not available. pub fn flush_linker_state(&self) -> anyhow::Result<()> { let streaming = self.streaming.as_ref().ok_or_else(|| { @@ -1760,8 +1781,10 @@ impl Unirust { streaming.flush_state(&persistence) } - /// Restore linker state from persistent storage. - /// Call this after enable_streaming() to recover cluster ID mappings. + /// Restore saved cluster-ID mappings after [`Self::initialize_streaming`]. + /// + /// Normal initialization already rebuilds resolution state from committed records. + /// This additionally loads mappings saved by [`Self::flush_linker_state`]. /// Returns the number of cluster_ids restored, or an error if persistence is not available. pub fn restore_linker_state(&mut self) -> anyhow::Result { // Restored IDs can differ from record-scan allocation order, including @@ -1780,9 +1803,10 @@ impl Unirust { streaming.restore_state(&persistence) } - /// Flush all linker state and sync to disk. - /// This is a convenience method that flushes linker state and ensures - /// all data is synced to disk. Call before shutdown for complete recovery. + /// Initialize the linker, flush its DSU buffers and ID-map snapshot, then + /// commit staged records and synchronize the persistent store. + /// + /// This synchronizes the current database; it does not create a separate backup. pub fn checkpoint_linker_state(&mut self) -> anyhow::Result<()> { self.initialize_streaming()?; let db = self @@ -1808,10 +1832,10 @@ impl Unirust { Ok(()) } - /// Flush the bounded amount of dirty state required for graceful process exit. + /// Flush dirty DSU state and staged records, then synchronize the store for exit. /// /// Full linker-map snapshots grow with total record count and are not needed by - /// the authoritative record-scan recovery path. + /// the authoritative record-scan recovery path. Work depends on pending dirty state. #[doc(hidden)] pub fn checkpoint_for_shutdown(&mut self) -> anyhow::Result<()> { self.initialize_streaming()?; diff --git a/src/linker.rs b/src/linker.rs index 7345d84..5f8346f 100644 --- a/src/linker.rs +++ b/src/linker.rs @@ -4,10 +4,10 @@ //! //! ## Parallelism Architecture //! -//! The linker uses a phased parallelism approach: -//! - **Phase 1 (Parallel)**: Extract key values and pre-compute candidates -//! - **Phase 2 (Sequential)**: Merge clusters in DSU (requires exclusive access) -//! - **Phase 3 (Parallel)**: Finalize cluster assignments +//! Batch linking extracts key values and strong-ID summaries in parallel. It then +//! processes records sequentially: find candidates, apply temporal guards and DSU +//! merges, and index each record before linking the next. Cluster mappings, member +//! lists, and guard summaries remain in memory even with persistent DSU/index backends. use crate::dsu::TemporalGuard; use crate::dsu::{Clusters, DsuBackend, MergeResult, TemporalDSU}; @@ -32,8 +32,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use tracing::{debug, instrument, warn}; -/// Type alias for inline candidate storage - avoids heap allocation for typical cases -/// 32 candidates covers 95%+ of queries based on production workload analysis +/// Store up to 32 candidates inline; larger candidate sets allocate on the heap. type CandidateVec = SmallVec<[(RecordId, Interval); 32]>; struct StoreLookup<'a>(&'a dyn RecordStore); @@ -252,8 +251,11 @@ struct ParallelExtractionResult { strong_id_summary: Option, } -/// Wrapper for linker state that can use either HashMap (unlimited) or LruCache (bounded). -/// This allows memory-bounded operation for billion-scale datasets. +/// A map wrapper supporting either retained HashMap entries or evicting LRU entries. +/// +/// The LRU variant has no durable spill path. [`StreamingLinker`] retains all +/// correctness-critical mappings and summaries in HashMaps, regardless of requested +/// linker-state cache limits. pub struct LinkerState { inner: LinkerStateInner, } @@ -421,23 +423,23 @@ pub struct StreamingLinker { dsu: DsuBackend, /// Index backend - can be in-memory or tiered identity_index: IndexBackend, - /// Cluster ID mappings (LRU-bounded when config provided) + /// Retained cluster ID mappings; size grows with resolution state. cluster_ids: LinkerState, next_cluster_id: u32, - /// Global cluster IDs for cross-shard tracking (LRU-bounded when config provided) + /// Retained global cluster IDs for cross-shard tracking. global_cluster_ids: LinkerState, /// Minimum durable record ID in each local cluster, independent of replay order. global_cluster_anchors: LinkerState, /// Shard ID for this linker (used in GlobalClusterId generation). shard_id: u16, - /// Strong ID summaries for conflict detection (LRU-bounded when config provided) + /// Retained strong ID summaries for conflict detection. strong_id_summaries: LinkerState, /// Root aliases and member lists let queries hydrate only candidate clusters. member_roots: FxHashMap, cluster_members: FxHashMap>, /// Use FxHashSet for faster hashing (non-cryptographic, perfect for internal keys) tainted_identity_keys: FxHashSet, - /// Record perspectives for same-perspective conflict detection (LRU-bounded when config provided) + /// Retained record perspectives for same-perspective conflict detection. record_perspectives: LinkerState, /// Use FxHashSet for faster hashing pending_keys: FxHashSet, @@ -487,7 +489,7 @@ impl StreamingLinker { } /// Initialize a streaming linker with a persistent DSU backend. - /// Use this for billion-scale deployments that need disk-backed DSU. + /// DSU entries use RocksDB and caches; other linker state still grows in memory. pub fn new_with_persistent_dsu( store: &dyn RecordStore, ontology: &Ontology, @@ -519,7 +521,7 @@ impl StreamingLinker { } /// Initialize a streaming linker with a tiered index backend. - /// Use this for billion-scale deployments that need tiered index. + /// Index buckets can spill to RocksDB; other linker state still grows in memory. pub fn new_with_tiered_index( store: &dyn RecordStore, ontology: &Ontology, @@ -1053,11 +1055,11 @@ impl StreamingLinker { /// /// This method uses a phased approach: /// - **Phase 1 (Parallel)**: Extract key values and build summaries for all records - /// - **Phase 2 (Sequential)**: Find candidates and merge clusters - /// - **Phase 3 (Sequential)**: Add records to index + /// - **Phase 2 (Sequential)**: For each record, find candidates, apply guarded + /// merges, and index the record before linking the next one /// - /// This provides significant speedup (30-50%) for large batches by parallelizing - /// the CPU-intensive extraction work. + /// Extraction runs in parallel; cluster mutations remain ordered. The benefit + /// depends on batch size, key complexity, and candidate overlap. #[instrument(skip(self, records, ontology), level = "debug")] pub fn link_records_batch_parallel( &mut self, @@ -2248,9 +2250,8 @@ mod tests { /// Identity key signature for linker deduplication. /// Distinct from sharding::IdentityKeySignature which uses a 32-byte hash. /// -/// Uses precomputed hash for O(1) hash lookups instead of re-hashing on every access. -/// This is critical for performance since LinkerKeySignature is used in hot-path -/// FxHashSet/FxHashMap operations. +/// Caches its hash to avoid re-hashing key values on every map access. Equality +/// still compares the entity type and key values when hashes collide. #[derive(Debug, Clone)] pub struct LinkerKeySignature { entity_type: String, diff --git a/src/model.rs b/src/model.rs index b3fbfee..c980fe7 100644 --- a/src/model.rs +++ b/src/model.rs @@ -7,9 +7,10 @@ use crate::temporal::Interval; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fmt; -// use string_interner::{DefaultBackend, StringInterner as ExternalStringInterner}; -/// Compact identifier for records +/// Compact record identifier, local to a store/shard. +/// Normal ingestion treats zero as an allocation request; explicit snapshot restore +/// can preserve zero. `u32::MAX` is reserved and cannot identify an ingested record. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] pub struct RecordId(pub u32); @@ -31,20 +32,20 @@ impl fmt::Display for ClusterId { /// Global cluster identifier for distributed entity resolution. /// -/// Encodes shard ownership, local cluster ID, and merge version in a single 64-bit value. +/// Encodes an originating shard, local numeric component, and version in 64 bits. /// Format: `(shard_id << 48) | (version << 32) | local_id` /// -/// This enables: -/// - Tracking which shard owns a cluster -/// - Detecting stale references after cross-shard merges -/// - Efficient comparison and hashing +/// Current distributed IDs use the minimum durable record ID in the local cluster +/// as their local component, independent of allocation-order [`ClusterId`] values. +/// Cross-shard merges are represented by durable redirects to a canonical global ID; +/// callers must resolve redirects rather than infer freshness from the version field. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct GlobalClusterId { - /// The shard that owns this cluster + /// The originating shard of this ID; redirects can point to a different shard. pub shard_id: u16, - /// Local cluster ID within the shard + /// Local numeric component; current distributed IDs use a durable record anchor. pub local_id: u32, - /// Merge version (incremented on cross-shard merges) + /// Encoded version component; cross-shard merges do not automatically increment it. pub version: u16, } @@ -58,7 +59,8 @@ impl GlobalClusterId { } } - /// Create from a local cluster ID on a specific shard + /// Copy a local cluster ID into the numeric component with version zero. + /// This conversion does not establish the durable record anchor used by the linker. pub fn from_local(shard_id: u16, local_id: ClusterId) -> Self { Self { shard_id, @@ -92,12 +94,14 @@ impl GlobalClusterId { Self::from_u64(u64::from_be_bytes(bytes)) } - /// Get the local cluster ID + /// Reinterpret the numeric local component as a [`ClusterId`]. + /// For current distributed IDs this component is a record anchor, so it need + /// not identify the corresponding cluster in an in-process linker. pub fn local_cluster_id(&self) -> ClusterId { ClusterId(self.local_id) } - /// Create a new version of this cluster (after a merge) + /// Copy this ID with the supplied version; does not perform a merge or redirect. pub fn with_new_version(&self, new_version: u16) -> Self { Self { shard_id: self.shard_id, @@ -106,7 +110,7 @@ impl GlobalClusterId { } } - /// Create with a new owner shard (for cross-shard merge) + /// Copy this ID with the supplied shard and version; does not move cluster data. pub fn with_new_owner(&self, new_shard_id: u16, new_version: u16) -> Self { Self { shard_id: new_shard_id, @@ -139,7 +143,8 @@ impl From for GlobalClusterId { } } -/// Compact identifier for attributes +/// Compact attribute identifier belonging to one store's interner. +/// Exchange attribute strings, not these IDs, between independently interned stores. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] pub struct AttrId(pub u32); @@ -149,7 +154,8 @@ impl fmt::Display for AttrId { } } -/// Compact identifier for values +/// Compact value identifier belonging to one store's interner. +/// Exchange value strings, not these IDs, between independently interned stores. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct ValueId(pub u32); diff --git a/src/ontology.rs b/src/ontology.rs index a48cb75..c1bcc9e 100644 --- a/src/ontology.rs +++ b/src/ontology.rs @@ -9,7 +9,8 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fmt; -/// An identity key defines which attributes must match for records to be considered the same entity +/// Attributes whose matching values generate entity-resolution candidates. +/// Temporal overlap and strong-identifier guards still determine whether records merge. #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct IdentityKey { /// The attributes that form the identity key (interned IDs for fast comparison) @@ -67,7 +68,8 @@ impl IdentityKey { self.attributes.is_empty() && !self.attribute_names.is_empty() } - /// Check if this identity key matches another over an overlapping interval + /// Compare interned attribute membership and count, ignoring order. + /// The interval parameter is unused; this does not compare values or time validity. pub fn matches(&self, other: &IdentityKey, _interval: Interval) -> bool { if self.attributes.len() != other.attributes.len() { return false; @@ -84,7 +86,8 @@ impl IdentityKey { } } -/// A strong identifier defines attributes that must not conflict within the same cluster +/// An attribute used to reject merges with overlapping, different strong-ID values +/// within the same perspective. Values from different perspectives may coexist. #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct StrongIdentifier { /// The attribute that serves as a strong identifier diff --git a/src/partitioned.rs b/src/partitioned.rs index a96b748..53a856a 100644 --- a/src/partitioned.rs +++ b/src/partitioned.rs @@ -1,24 +1,25 @@ //! # Partitioned Processing Module //! -//! Implements partition-local processing to eliminate lock contention. -//! Each partition owns its data exclusively - no locks needed within a partition. +//! In-memory partition-local processing. The parallel wrapper locks each partition +//! during its batch, allowing independent partitions to run concurrently. Persistent +//! shard services use a separate durable engine path, not these in-memory stores. //! //! ## Architecture //! //! ```text //! ┌─────────────────────────────────────────┐ -//! │ Router (Consistent Hash) │ +//! │ Partition Hash Dispatch │ //! └─────────────────────────────────────────┘ //! │ │ │ //! ┌─────────┴─────────┴─────────┴─────────┐ //! ▼ ▼ ▼ //! Partition 0 Partition 1 Partition N -//! (exclusive) (exclusive) (exclusive) +//! (batch lock) (batch lock) (batch lock) //! └───────────────────┴───────────────────┘ //! │ //! ┌─────────▼─────────┐ //! │ Merge Coordinator │ -//! │ (async channel) │ +//! │ (bounded channel) │ //! └───────────────────┘ //! ``` @@ -285,7 +286,8 @@ impl Partition { } /// Optimized batch processing with parallel extraction. - /// Uses link_records_batch_parallel for 3x throughput improvement. + /// Uses `link_records_batch_parallel` for parallel extraction and ordered, + /// temporally guarded linking. Throughput depends on the workload. /// /// Phase 1: Batch add all records to store /// Phase 2: Parallel extraction of key values and summaries @@ -584,9 +586,9 @@ pub struct PartitionedUnirust { total_records: AtomicU64, } -/// Thread-safe partitioned Unirust with per-partition locks for TRUE parallel processing +/// In-memory engine with parallel batch dispatch and per-partition locks. pub struct ParallelPartitionedUnirust { - /// Each partition has its own Mutex - no global lock contention! + /// Each partition has its own mutex; work routed to that partition is serialized. partitions: Vec>, /// Configuration config: PartitionConfig, @@ -710,9 +712,8 @@ impl PartitionedUnirust { // Phase 1: Partition records by primary key let partitioned = self.partition_records(records); - // Phase 2: Process each partition in parallel - // This is where we get the speedup - each partition processes independently - // We need to use indices because we can't parallelize over &mut references directly + // Phase 2: This wrapper processes partitions sequentially. + // ParallelPartitionedUnirust provides concurrent partition dispatch. let ontology = &self.ontology; // Collect results from each partition @@ -812,12 +813,7 @@ impl PartitionedUnirust { } } -/// Wrapper that provides lock-free access to PartitionedUnirust -/// using interior mutability via UnsafeCell -/// -/// SAFETY: This is safe because: -/// 1. Each partition is only accessed by one thread at a time (partitioning ensures this) -/// 2. Cross-partition communication uses thread-safe channels +/// Synchronize access to [`PartitionedUnirust`] using a reader/writer lock. pub struct PartitionedUnirustHandle { inner: parking_lot::RwLock, } @@ -829,7 +825,7 @@ impl PartitionedUnirustHandle { } } - /// Ingest a batch with write lock (still faster than single-instance due to partitioning) + /// Ingest a batch while holding the wrapper's write lock. pub fn ingest_batch(&self, records: Vec<(u32, Record)>) -> Vec { self.inner.write().ingest_batch(records) } @@ -999,8 +995,8 @@ impl ParallelPartitionedUnirust { } } - /// Ingest a batch of records using TRUE parallel partition processing. - /// Each partition is processed independently with its own lock - no global contention! + /// Dispatch a batch across partitions using Rayon. + /// Each nonempty partition is processed under its own mutex. #[instrument(skip(self, records), level = "debug")] pub fn ingest_batch(&self, records: Vec<(u32, Record)>) -> Result> { if records.is_empty() { @@ -1015,7 +1011,7 @@ impl ParallelPartitionedUnirust { let partitioned = self.partition_records(records); // Phase 2: Process ALL partitions in PARALLEL using rayon - // Each partition has its own Mutex, so no global lock contention! + // Independent partitions can run concurrently; each mutex serializes its batch. let ontology = &self.ontology; let all_results: Result>> = partitioned diff --git a/src/perf/async_wal.rs b/src/perf/async_wal.rs index ff7ffcc..95f4193 100644 --- a/src/perf/async_wal.rs +++ b/src/perf/async_wal.rs @@ -1,9 +1,10 @@ -//! Async Write-Ahead Log with coalescing +//! Experimental background batch-file writer. //! -//! Removes WAL fsync from the critical path by: -//! 1. Submitting writes to a background thread -//! 2. Coalescing multiple writes into single fsync -//! 3. Optional io_uring support for async disk I/O +//! Submissions are coalesced by a background thread using blocking file I/O. +//! Each flush replaces the file with the latest batch, and write/sync errors are +//! not returned through tickets. Ticket completion therefore does not prove durable +//! delivery. This utility is not the production distributed ingest WAL and must not +//! be used as an append-only recovery log. use std::fs::{self, OpenOptions}; use std::io::Write; @@ -24,7 +25,7 @@ pub struct AsyncWalConfig { pub max_coalesce_records: usize, /// Channel capacity for write requests pub channel_capacity: usize, - /// Whether to actually sync to disk (false for testing) + /// Whether to attempt file synchronization; errors are not reported to tickets. pub sync_enabled: bool, } @@ -82,7 +83,8 @@ impl WalTicket { } } - /// Check if the write has been synced to disk + /// Check whether the writer finished processing the batch containing this ticket. + /// Completion does not report I/O success or guarantee synchronization. #[inline] pub fn is_complete(&self) -> bool { self.completed.load(Ordering::Acquire) @@ -112,7 +114,7 @@ impl WalTicket { } } -/// Async Write-Ahead Log +/// Experimental coalescing batch writer; see module-level durability limitations. pub struct AsyncWal { /// Channel to send write requests tx: Sender, diff --git a/src/perf/atomic_dsu.rs b/src/perf/atomic_dsu.rs index 9a71910..5a6de85 100644 --- a/src/perf/atomic_dsu.rs +++ b/src/perf/atomic_dsu.rs @@ -1,12 +1,8 @@ //! # Lock-Free Atomic DSU (Disjoint Set Union) //! -//! High-performance union-find implementation using only atomic operations. -//! No locks required - all operations use CAS (Compare-And-Swap). -//! -//! ## Performance Characteristics -//! - find(): O(α(n)) amortized with path compression -//! - union(): O(α(n)) amortized with union-by-rank -//! - No lock contention under high concurrency +//! Union-find using atomic parent/rank storage, compare-and-swap merges, and +//! best-effort path compression. Contended updates can retry. This standalone +//! utility does not implement the temporal guards required by the entity linker. //! //! ## Safety //! Uses atomic operations with appropriate memory ordering: @@ -16,7 +12,7 @@ use crate::model::RecordId; use std::sync::atomic::{AtomicU32, AtomicU8, Ordering}; -/// Maximum number of records supported (configurable at compile time) +/// Default number of slots; callers can supply a different capacity. const MAX_RECORDS: usize = 16_777_216; // 16M records per partition /// Lock-free DSU with atomic parent pointers diff --git a/src/perf/bigtable_opts.rs b/src/perf/bigtable_opts.rs index 1c39cb7..7affa5f 100644 --- a/src/perf/bigtable_opts.rs +++ b/src/perf/bigtable_opts.rs @@ -28,16 +28,16 @@ use std::time::Instant; /// Bloom filter optimized for identity key lookups. /// Reduces disk/index reads by filtering out keys that definitely don't exist. /// -/// From Bigtable paper: "We also allow clients to create Bloom filters for -/// SSTables in a particular locality group. A Bloom filter allows us to ask -/// whether an SSTable might contain any data for a specified row/column pair." +/// The current insertion implementation uses an unsynchronized raw-pointer cast +/// to mutate shared storage. It has unresolved interior-mutability safety issues +/// and is not suitable for concurrent mutation. #[derive(Debug)] pub struct IdentityBloomFilter { /// The underlying bloom filter filter: BloomFilter, /// Number of keys inserted key_count: AtomicU64, - /// False positive rate estimate (updated periodically) + /// Fixed initial false-positive estimate; currently not updated from occupancy. estimated_fpr: AtomicU64, // Stored as fpr * 1_000_000 } @@ -52,12 +52,12 @@ impl IdentityBloomFilter { } } - /// Create with default 64KB size (~500K keys at 1% FPR) + /// Create with 64 KiB of bit storage. False-positive rate grows with inserted keys. pub fn default_size() -> Self { Self::new(64) } - /// Create with large 256KB size (~2M keys at 1% FPR) + /// Create with 256 KiB of bit storage. False-positive rate grows with inserted keys. pub fn large() -> Self { Self::new(256) } @@ -65,8 +65,8 @@ impl IdentityBloomFilter { /// Insert an identity key signature #[inline] pub fn insert(&self, signature: &IdentityKeySignature) { - // SAFETY: BloomFilter insert only sets bits, never clears them. - // Concurrent inserts may set the same bits multiple times but this is safe. + // This cast does not provide interior mutability or synchronize Vec + // updates. Setting bits monotonically is not a sufficient safety argument. unsafe { let filter_ptr = &self.filter as *const BloomFilter as *mut BloomFilter; (*filter_ptr).insert(signature); @@ -87,7 +87,7 @@ impl IdentityBloomFilter { self.key_count.load(Ordering::Relaxed) } - /// Estimate current false positive rate + /// Return the initial estimate (0.001); this is not a measured or updated rate. pub fn estimated_fpr(&self) -> f64 { self.estimated_fpr.load(Ordering::Relaxed) as f64 / 1_000_000.0 } @@ -117,7 +117,7 @@ pub struct ScanCacheEntry { /// In our case, we cache the candidate records for each identity key signature, /// avoiding repeated index scans for hot keys. /// -/// Uses RocksDB-inspired sharded LRU for 4-8x concurrent throughput improvement. +/// Uses a sharded LRU to distribute cache operations across independent locks. pub struct ScanCache { /// Sharded LRU cache mapping identity key signatures to candidates /// Using 16 shards for good parallelism without overhead diff --git a/src/perf/compression.rs b/src/perf/compression.rs index 2332654..be7f7c1 100644 --- a/src/perf/compression.rs +++ b/src/perf/compression.rs @@ -1,27 +1,19 @@ //! # LZ4 Compression Module //! -//! High-speed compression for records and WAL entries. +//! LZ4 block compression helpers with a four-byte uncompressed-size prefix. //! -//! From Bigtable paper (Section 6): "Many clients use a two-pass custom -//! compression scheme. The first pass uses Bentley and McIlroy's scheme, -//! which compresses long common strings across a large window. The second -//! pass uses a fast compression algorithm that looks for repetitions in -//! a small 16 KB window of the data." -//! -//! LZ4 provides similar characteristics: -//! - Encode speed: 780+ MB/s -//! - Decode speed: 4970+ MB/s -//! - Compression ratio: ~2.1x for typical data +//! Compression ratio and throughput depend on the data, batch size, and hardware. use std::io; -/// Compression level for LZ4 +/// Reserved compression-level labels; the helpers currently use the same LZ4 +/// implementation and do not select an algorithm based on this enum. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum CompressionLevel { - /// Fastest compression (default) + /// Default label. #[default] Fast, - /// Higher compression ratio (slower) + /// Reserved higher-compression label; not implemented as a separate mode. High, } @@ -44,14 +36,14 @@ pub fn decompress(compressed: &[u8]) -> io::Result> { .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) } -/// Compress data with a known maximum size hint for better performance. +/// Compress data; the size hint is currently ignored. #[inline] pub fn compress_with_hint(data: &[u8], _size_hint: usize) -> Vec { // lz4_flex handles buffer sizing internally compress(data) } -/// Decompress with a known output size for better performance. +/// Decompress using the supplied output size when it matches the stored prefix. #[inline] pub fn decompress_with_size(compressed: &[u8], uncompressed_size: usize) -> io::Result> { // Skip the size prefix if present diff --git a/src/perf/interner.rs b/src/perf/interner.rs index a0d8181..40527f5 100644 --- a/src/perf/interner.rs +++ b/src/perf/interner.rs @@ -1,14 +1,14 @@ //! # Concurrent Interner //! -//! Thread-safe string interning using DashMap for lock-free concurrent access. -//! Eliminates the need for write locks in the hot path. +//! Thread-safe string interning using DashMap's sharded locks and atomic ID counters. +//! Operations can contend on a map shard; there is no single lock around the interner. use crate::model::{AttrId, InternerLookup, ValueId}; use dashmap::DashMap; use std::sync::atomic::{AtomicU32, Ordering}; /// Thread-safe concurrent interner for attributes and values. -/// Uses DashMap for lock-free concurrent access. +/// Uses DashMap's shard-level synchronization for concurrent access. pub struct ConcurrentInterner { /// Attribute string to ID mapping attr_to_id: DashMap, @@ -38,7 +38,7 @@ impl ConcurrentInterner { } /// Intern an attribute string, returning its ID. - /// Thread-safe and lock-free. + /// Thread-safe; may acquire a DashMap shard lock. #[inline] pub fn intern_attr(&self, attr: &str) -> AttrId { // Fast path: check if already interned @@ -61,7 +61,7 @@ impl ConcurrentInterner { } /// Intern a value string, returning its ID. - /// Thread-safe and lock-free. + /// Thread-safe; may acquire a DashMap shard lock. #[inline] pub fn intern_value(&self, value: &str) -> ValueId { // Fast path: check if already interned diff --git a/src/perf/mod.rs b/src/perf/mod.rs index 5276dc6..391625f 100644 --- a/src/perf/mod.rs +++ b/src/perf/mod.rs @@ -1,14 +1,18 @@ //! # Performance Optimizations Module //! -//! Low-latency optimizations for maximum throughput: -//! - Lock-free DSU with atomic parent links -//! - SIMD-accelerated key hashing -//! - Zero-copy record passing +//! Performance utilities with different integration and durability requirements: +//! - Atomic parent links for a standalone DSU without temporal guards +//! - Scalar and platform-specific batch hashing +//! - Shared record slices //! - Cache-line aligned metrics //! - Lock-free ingest queue -//! - Async WAL with write coalescing -//! - Concurrent string interning +//! - Experimental background batch-file writing +//! - String interning with sharded locks //! - Bigtable-inspired caching (bloom filters, scan cache, block cache) +//! +//! These helpers are not all used by the production ingest path. In particular, +//! [`AsyncWal`] is not the durable distributed ingest WAL, and [`AtomicDSU`] does +//! not replace the temporally guarded linker DSU. pub mod aligned; pub mod async_wal; diff --git a/src/perf/queue.rs b/src/perf/queue.rs index c735e36..6445ddf 100644 --- a/src/perf/queue.rs +++ b/src/perf/queue.rs @@ -1,7 +1,8 @@ -//! Lock-free ingest queue for high-throughput throughput +//! Bounded ingest-job queue and batch aggregation utilities. //! -//! Replaces RwLock contention with bounded lock-free MPSC queue. -//! Single aggregator thread processes batches with exclusive access. +//! Uses a lock-free ArrayQueue, which supports multiple producers and consumers. +//! A caller can use a single aggregator to collect jobs for exclusive processing. +//! Queue capacity bounds jobs, not the records or bytes retained by each job. use crossbeam_queue::ArrayQueue; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; @@ -12,10 +13,11 @@ use tokio::sync::oneshot; /// Capacity for the lock-free ingest queue pub const QUEUE_CAPACITY: usize = 10_000; -/// Maximum records per aggregated batch +/// Record-count threshold at which an aggregator requests a flush. +/// Whole jobs can exceed this threshold; it is not a hard batch-size limit. pub const MAX_BATCH_SIZE: usize = 5000; -/// Maximum wait time before flushing partial batch +/// Elapsed-time threshold checked by the aggregator; callers drive polling/flushing. pub const MAX_BATCH_WAIT: Duration = Duration::from_micros(2000); /// A job submitted to the ingest queue diff --git a/src/perf/sharded_cache.rs b/src/perf/sharded_cache.rs index 036ddfb..e1af22e 100644 --- a/src/perf/sharded_cache.rs +++ b/src/perf/sharded_cache.rs @@ -2,17 +2,11 @@ //! //! RocksDB-inspired sharded LRU cache for high-concurrency workloads. //! -//! From RocksDB Tuning Guide: -//! "Both LRUCache and ClockCache are sharded to mitigate lock contention. -//! Each shard maintains its own LRU list and hash table. Synchronization -//! is done via a per-shard mutex." +//! Hash-based shards maintain independent LRU lists and reader/writer locks. +//! Operations on the same shard still contend; throughput depends on key distribution. //! -//! Benefits: -//! - 4-8x throughput improvement on concurrent cache operations -//! - Reduced lock contention via hash-based sharding -//! - Per-shard statistics for monitoring -//! -//! Default: 16 shards (2^4), minimum 512KB per shard +//! Defaults use 16 shards and a minimum of 1024 entries per shard. Capacities count +//! entries, not bytes; the per-shard minimum can exceed the requested total capacity. use lru::LruCache; use parking_lot::RwLock; @@ -23,12 +17,12 @@ use std::sync::atomic::{AtomicU64, Ordering}; /// Configuration for sharded cache #[derive(Debug, Clone)] pub struct ShardedCacheConfig { - /// Total capacity across all shards + /// Requested total entry capacity; per-shard minimums may increase the actual total. pub total_capacity: usize, /// Number of shard bits (shards = 2^shard_bits) /// Default: 4 (16 shards) pub shard_bits: u8, - /// Minimum capacity per shard (prevents thrashing) + /// Minimum entry capacity per shard /// Default: 1024 entries pub min_shard_capacity: usize, } @@ -48,12 +42,12 @@ impl ShardedCacheConfig { pub fn high_throughput(capacity: usize) -> Self { Self { total_capacity: capacity, - shard_bits: 6, // 64 shards for maximum parallelism + shard_bits: 6, // 64 independently locked shards min_shard_capacity: 512, } } - /// Memory-efficient configuration with fewer shards + /// Configuration with fewer shards and a larger per-shard minimum. pub fn memory_efficient(capacity: usize) -> Self { Self { total_capacity: capacity, diff --git a/src/perf/simd_hash.rs b/src/perf/simd_hash.rs index b32cdee..4dc11af 100644 --- a/src/perf/simd_hash.rs +++ b/src/perf/simd_hash.rs @@ -1,22 +1,20 @@ -//! # SIMD-Accelerated Hashing +//! # Scalar and Platform-Specific Hashing //! -//! High-performance batch hashing using SIMD instructions. -//! Falls back to scalar FxHash on platforms without SIMD support. -//! -//! ## Performance -//! - AVX2: ~8x throughput for batch hashing -//! - NEON: ~4x throughput on ARM -//! - Scalar fallback: Same as rustc_hash::FxHasher +//! Fx-style scalar mixing with an AVX2 batch path selected at compile time on +//! x86_64 builds enabling that feature. The byte-batch API uses scalar hashing on +//! other builds. These hashes are not a portable storage or wire-format contract; +//! platform paths are not guaranteed to produce identical outputs. Throughput is +//! workload- and hardware-dependent. use std::hash::Hasher; -/// SIMD-accelerated batch hasher +/// Scalar hasher with a platform-specific byte-batch helper. pub struct SimdHasher { state: u64, } impl SimdHasher { - /// FxHash constant (good mixing properties) + /// Multiplicative mixing constant used by this implementation. const K: u64 = 0x517cc1b727220a95; /// Create a new hasher diff --git a/src/perf/write_buffer.rs b/src/perf/write_buffer.rs index 177ffdf..3677905 100644 --- a/src/perf/write_buffer.rs +++ b/src/perf/write_buffer.rs @@ -2,16 +2,15 @@ //! //! RocksDB-inspired write buffer management with backpressure. //! -//! From RocksDB Write Buffer Manager: -//! "Helps manage total memory consumed by memtables across multiple -//! column families and DB instances, preventing memory bloat during -//! heavy write loads." -//! //! Key features: -//! - Memory limit enforcement across multiple buffers +//! - Accounting for caller-reported buffer reservations //! - Automatic flush triggers at configurable thresholds //! - Backpressure/stalling when memory exceeds limits //! - Statistics for monitoring memory pressure +//! +//! This is not RocksDB's native write-buffer manager or a hard process-memory cap. +//! Concurrent reservations can overshoot the threshold, and accounting depends on +//! callers reporting allocations and completing flushes. use parking_lot::{Mutex, RwLock}; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; @@ -21,7 +20,7 @@ use std::time::{Duration, Instant}; /// Configuration for write buffer manager #[derive(Debug, Clone)] pub struct WriteBufferConfig { - /// Total memory limit for all buffers (bytes) + /// Reservation threshold in bytes; concurrent reservations can exceed it. pub memory_limit: usize, /// Trigger flush when mutable buffer reaches this ratio of limit /// Default: 0.9 (90%) @@ -32,7 +31,7 @@ pub struct WriteBufferConfig { /// Allow stalling writers when memory exceeded /// Default: true pub allow_stall: bool, - /// Maximum stall duration before forcing through + /// Maximum stall duration before rejecting the reservation /// Default: 1 second pub max_stall_duration: Duration, /// Check interval for memory pressure diff --git a/src/perf/zero_copy.rs b/src/perf/zero_copy.rs index a11a942..1aae255 100644 --- a/src/perf/zero_copy.rs +++ b/src/perf/zero_copy.rs @@ -1,11 +1,11 @@ //! # Zero-Copy Record Passing //! -//! Eliminates unnecessary allocations and copies in the hot path. +//! Share immutable record slices after constructing their backing allocation. //! //! ## Key Optimizations //! - `RecordSlice`: Shared ownership of record batches via Arc -//! - `ZeroCopyBatch`: Pre-partitioned records without cloning -//! - Inline storage for small batches +//! - `ZeroCopyBatch`: Clones records into partition order, then shares partition slices +//! - `SmallBatch`: Inline slots for up to eight records; record payloads may allocate use crate::model::{Record, RecordId}; use std::ops::Deref; @@ -87,16 +87,16 @@ impl Deref for RecordSlice { } } -/// A batch of records pre-partitioned for zero-copy processing +/// A batch that shares partition slices after cloning records into partition order. pub struct ZeroCopyBatch { - /// Original records (shared) + /// Reordered records (shared) records: RecordSlice, /// Partition assignments: (partition_id, start_idx, len) partitions: Vec<(usize, usize, usize)>, } impl ZeroCopyBatch { - /// Create a new batch and partition records + /// Clone records into contiguous partitions and create shared backing storage. /// /// The partition function receives a record and returns its partition ID. pub fn new(records: Vec, partition_count: usize, partition_fn: F) -> Self diff --git a/src/persistence.rs b/src/persistence.rs index 3e4924d..4860cdf 100644 --- a/src/persistence.rs +++ b/src/persistence.rs @@ -1,3 +1,11 @@ +//! RocksDB record storage, durable interner mappings, and checkpoint utilities. +//! +//! Records and metadata use binary encodings. High-level engine ingestion performs +//! entity resolution before committing staged records and synchronizing the store. +//! Direct store insertion APIs provide storage only; they do not run entity resolution. +//! Recovery reconstructs derived linker state from committed records. Coordinated +//! cluster checkpoints also capture shard/WAL state through the distributed protocol. + use crate::model::{GlobalClusterId, Record, RecordId, RecordIdentity, StringInterner}; use crate::store::{ records_have_same_payload, RecordStore, SourceRecordReservation, SourceReservationError, Store, @@ -502,14 +510,14 @@ const DEFAULT_LEVEL_BASE_MB: u64 = 512; const DEFAULT_BLOOM_BITS_PER_KEY: f64 = 10.0; const DEFAULT_MEMTABLE_PREFIX_BLOOM_RATIO: f64 = 0.1; // Aggressive compaction deferral: favor ingest throughput over compaction -// Default 20 MB/s rate limit ensures compaction doesn't starve ingest +// Default 20 MB/s rate limit trades compaction progress against foreground I/O. const DEFAULT_RATE_LIMIT_MBPS: u64 = 20; // Reduce compaction threads to minimize CPU contention with ingest const DEFAULT_COMPACTION_THREADS: i32 = 1; -// Keep flush threads higher to avoid write stalls +// Reserve more flush threads than compaction threads; write stalls can still occur. const DEFAULT_FLUSH_THREADS: i32 = 2; -// Disable auto compaction by default for maximum ingest throughput -// Compaction runs during quiet periods or can be triggered manually +// Automatic compaction is enabled by default. Disabling it requires explicit tuning +// and can accumulate SST files and pending compaction work during sustained ingest. const DEFAULT_DISABLE_AUTO_COMPACTION: bool = false; // Soft limit before slowing writes (4GB default - very permissive) const DEFAULT_SOFT_PENDING_COMPACTION_GB: u64 = 4; @@ -533,6 +541,10 @@ struct StorageManifest { app_version: String, } +/// RocksDB-backed records with caches for committed data and retained pending records. +/// +/// Staged records remain in memory until committed or discarded, so record-cache +/// capacity does not bound pending-batch memory or the engine's full resolution state. pub struct PersistentStore { inner: Store, db: Arc, @@ -562,6 +574,9 @@ pub struct PersistentOpenOptions { } impl PersistentStore { + /// Open or create a persistent store with automatic repair disabled. + /// Opening the store does not initialize entity resolution; attach it to an engine + /// with [`crate::Unirust::with_store`] and initialize or ingest through that engine. pub fn open(path: impl AsRef) -> Result { Self::open_with_options(path, PersistentOpenOptions::default()) } @@ -2941,7 +2956,8 @@ pub mod index_encoding { use serde::{Deserialize, Serialize}; /// Compact bucket format for warm/cold tier storage - /// Uses 16 bytes per interval vs 32+ for full IntervalTree node + /// Each interval tuple contains a u32 record ID and two i64 endpoints. Serialized + /// size and in-memory layout differ; the enclosing vectors also carry overhead. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CompactBucketData { /// Record intervals: (record_id, start, end) diff --git a/src/sharding.rs b/src/sharding.rs index d93e122..1112e6b 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -88,7 +88,7 @@ impl BloomFilter { let num_u64s = size_bytes / 8; Self { bits: vec![0u64; num_u64s.max(1)], - num_hashes: 7, // Optimal for ~1% false positive rate + num_hashes: 7, // False-positive rate depends on bits per inserted key. } } @@ -550,9 +550,9 @@ impl ReconciliationCandidates { /// Incremental reconciler for cross-shard cluster merges. /// -/// Instead of loading all records from all shards (O(n)), -/// this uses boundary indices to find and merge only clusters -/// that share identity keys (O(k) where k = boundary keys). +/// Uses boundary indices to find clusters that share identity keys without loading +/// every shard's records. Work depends on signature count and candidate cluster pairs; +/// many clusters sharing one key can still require pairwise comparisons. #[derive(Debug)] pub struct IncrementalReconciler { /// Boundary indices from all shards. diff --git a/src/store.rs b/src/store.rs index 44d7e37..c615c08 100644 --- a/src/store.rs +++ b/src/store.rs @@ -66,7 +66,8 @@ pub trait RecordStore: Send + Sync { /// Get a record by ID. fn get_record(&self, id: RecordId) -> Option; - /// Get a reference to a record by ID (avoids cloning). Default uses get_record. + /// Borrow a record when the backend can lend one without cloning. + /// The default returns `None`; callers can fall back to [`Self::get_record`]. fn get_record_ref(&self, _id: RecordId) -> Option<&Record> { None // Default implementation returns None, callers should use get_record } diff --git a/src/temporal.rs b/src/temporal.rs index 801ac76..4dca854 100644 --- a/src/temporal.rs +++ b/src/temporal.rs @@ -84,7 +84,8 @@ impl Interval { self.start <= instant && instant < self.end } - /// Check if this interval is empty (should never happen with our validation) + /// Check whether start is at or after end. [`Self::new`] rejects such intervals, + /// but the public fields also permit constructing them directly. pub fn is_empty(&self) -> bool { self.start >= self.end }