From f82bdcd15197381a0a2b1b4256258e14a3f9d41e Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Fri, 14 Aug 2026 10:14:37 -0700 Subject: [PATCH 1/6] feat(rust)!: extract prebuilt fullmap archive in Rust (streaming zstd+tar, GIL-free) [US-001] --- rust/Cargo.lock | 79 ++++++ rust/Cargo.toml | 7 + rust/src/fullmap.rs | 437 ++++++++++++++++++++++++++++++++- rust/src/lib.rs | 5 +- rust/tests/build_golden.rs | 196 +-------------- rust/tests/common/mod.rs | 210 ++++++++++++++++ rust/tests/extract_prebuilt.rs | 123 ++++++++++ src/tablassert/rs.pyi | 1 + 8 files changed, 868 insertions(+), 190 deletions(-) create mode 100644 rust/tests/common/mod.rs create mode 100644 rust/tests/extract_prebuilt.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index ffd9ab9..b9f3fad 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -45,6 +45,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -127,6 +129,16 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -216,6 +228,16 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + [[package]] name = "js-sys" version = "0.3.103" @@ -304,6 +326,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + [[package]] name = "portable-atomic" version = "1.13.1" @@ -541,9 +569,22 @@ dependencies = [ "rustc-hash", "serde", "serde_json", + "tar", "tempfile", "uuid", "xxhash-rust", + "zstd", +] + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", ] [[package]] @@ -654,6 +695,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "xxhash-rust" version = "0.8.17" @@ -671,3 +722,31 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 14e1c35..83f6b72 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -37,8 +37,13 @@ rlimit = "0.10" rustc-hash = "1" serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["preserve_order"] } +tar = "0.4" uuid = { version = "1", features = ["v3"] } xxhash-rust = { version = "0.8", features = ["xxh64", "xxh3"] } +# Streaming zstd decompression for the prebuilt `fullmap.tar.zst` archive +# (`extract_prebuilt_fullmap`); the C-backed decoder is the fastest option and +# `cc` is available in the build environment. +zstd = "0.13" [dev-dependencies] bincode = "1" @@ -46,7 +51,9 @@ flate2 = { version = "1", features = ["zlib-rs"], default-features = false } redb = { git = "https://github.com/cberner/redb.git", rev = "a35e7cc86f191d08a444d5973469b34673d586fc", features = ["experimental_cursor"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +tar = "0.4" tempfile = "3" +zstd = "0.13" [lints.rust] unused_imports = "deny" diff --git a/rust/src/fullmap.rs b/rust/src/fullmap.rs index f816cd9..8d14abe 100644 --- a/rust/src/fullmap.rs +++ b/rust/src/fullmap.rs @@ -10,16 +10,17 @@ use rustc_hash::FxHasher; use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::cmp::Reverse; -use std::collections::{BinaryHeap, HashMap, HashSet}; +use std::collections::{BTreeMap, BinaryHeap, HashMap, HashSet}; use std::fs::File; use std::hash::{BuildHasher, BuildHasherDefault}; use std::io::{BufRead, BufReader, BufWriter, Read, Write}; -use std::os::unix::fs::MetadataExt; +use std::os::unix::fs::{MetadataExt, PermissionsExt}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{sync_channel, Receiver, SyncSender}; use std::sync::{Arc, Mutex, OnceLock, RwLock}; use std::thread; +use tar::EntryType; use xxhash_rust::xxh3::xxh3_128; use xxhash_rust::xxh64::xxh64; @@ -2461,6 +2462,438 @@ fn validate_schema(database: &ReadOnlyDatabase) -> PyResult<()> { } } +// --------------------------------------------------------------------------- +// Prebuilt archive extraction +// --------------------------------------------------------------------------- + +/// Size of the file buffer feeding the zstd decoder during prebuilt archive +/// extraction (~8 MiB): one large sequential read per chunk keeps the multi-GB +/// decompression IO-bound instead of syscall-bound (the same capacity +/// `RunWriter` uses for its spill writes). +const EXTRACT_READ_BUFFER_BYTES: usize = 8 * 1024 * 1024; + +/// Best-effort `(detail: str)` progress report for `extract_prebuilt_fullmap`. +/// The extraction runs with the GIL released, so the GIL is re-acquired briefly +/// per call and the callback result is deliberately discarded — a failing +/// Python callback must never kill the extraction (mirrors `Progress::call`). +fn extract_report(progress: &Option>, detail: &str) { + if let Some(cb) = progress { + Python::attach(|py| { + let _ = cb.call1(py, (detail,)); + }); + } +} + +/// Validated RELATIVE member path for one archive entry. +/// +/// `tar::Entry::path()` returns the header path AS-IS — tar 0.4's own +/// traversal protection lives in `Entry::unpack_in`, which SILENTLY SKIPS +/// escaping members instead of failing. The contract here is to ERROR loudly, +/// so the check is explicit: every component must be a plain name (`..`, +/// absolute roots, and drive prefixes all fail the whole extraction). +fn validated_entry_path(entry: &tar::Entry<'_, R>) -> PyResult { + use std::path::Component; + + let raw = entry.path().map_err(|e| { + py_err(format!( + "prebuilt archive has an entry with an unreadable path: {e}" + )) + })?; + let mut rel = PathBuf::new(); + for component in raw.components() { + match component { + Component::CurDir => {} // `./` is harmless; normalize it away + Component::Normal(part) => rel.push(part), + other => { + return Err(py_err(format!( + "prebuilt archive entry {} has an unsafe path ({other:?}); \ + entries must be relative with no '..' components", + raw.display() + ))); + } + } + } + if rel.as_os_str().is_empty() { + return Err(py_err(format!( + "prebuilt archive entry {} has an empty path", + raw.display() + ))); + } + Ok(rel) +} + +/// The shard index encoded in a `.s.redb` file name (matches +/// `\.s(\d+)\.redb$`), or `None` for a non-shard `.redb` (primary candidate). +fn shard_index_of_name(file_name: &str) -> Option { + let stem = file_name.strip_suffix(".redb")?; + let marker = stem.rfind(".s")?; + let digits = &stem[marker + 2..]; + if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + digits.parse().ok() +} + +/// Recursively collect the `.redb` files under `dir` (archive members may be +/// nested in subdirectories), classifying each as a shard (`s` -> index) or +/// a primary candidate. A duplicate shard index is an error, never a +/// silent overwrite. +fn scan_extracted_redb( + dir: &Path, + shards: &mut BTreeMap, + primaries: &mut Vec, +) -> PyResult<()> { + let read_dir = std::fs::read_dir(dir).map_err(|e| { + py_err(format!( + "failed to scan extracted prebuilt archive dir {}: {e}", + dir.display() + )) + })?; + for item in read_dir { + let item = item.map_err(py_err)?; + let path = item.path(); + let file_type = item.file_type().map_err(py_err)?; + if file_type.is_dir() { + scan_extracted_redb(&path, shards, primaries)?; + } else if file_type.is_file() && path.extension().and_then(|x| x.to_str()) == Some("redb") { + let name = item.file_name().to_string_lossy().into_owned(); + match shard_index_of_name(&name) { + Some(index) => { + if shards.contains_key(&index) { + return Err(py_err(format!( + "prebuilt archive contains a duplicate shard s{index} at {}", + path.display() + ))); + } + shards.insert(index, path); + } + None => primaries.push(path), + } + } + } + Ok(()) +} + +/// Stream-extract, validate, and atomically rename one prebuilt fullmap +/// archive (the shared tail of `extract_prebuilt_fullmap`; see its doc for the +/// full contract). On success the temp dir is removed last; the caller cleans +/// it up on error. +fn extract_validate_rename( + archive: &Path, + temp_dir: &Path, + output: &Path, + progress: &Option>, +) -> PyResult<()> { + // Stream File -> 8 MiB BufReader -> zstd decoder -> tar entries; the + // decompressed tar is NEVER materialized on disk. + let file = File::open(archive).map_err(|e| { + py_err(format!( + "failed to open prebuilt archive {}: {e}", + archive.display() + )) + })?; + let reader = BufReader::with_capacity(EXTRACT_READ_BUFFER_BYTES, file); + let decoder = zstd::Decoder::new(reader).map_err(|e| { + py_err(format!( + "prebuilt archive {} is not valid zstd: {e}", + archive.display() + )) + })?; + let mut tar_archive = tar::Archive::new(decoder); + let entries = tar_archive.entries().map_err(|e| { + py_err(format!( + "failed to read tar entries from prebuilt archive {}: {e}", + archive.display() + )) + })?; + + for entry in entries { + let mut entry = entry.map_err(|e| { + py_err(format!( + "failed to read an entry of prebuilt archive {}: {e}", + archive.display() + )) + })?; + let rel = validated_entry_path(&entry)?; + extract_report(progress, &format!("extracting {}", rel.display())); + let dest = temp_dir.join(&rel); + match entry.header().entry_type() { + EntryType::Directory => { + std::fs::create_dir_all(&dest).map_err(|e| { + py_err(format!( + "failed to create directory {} from prebuilt archive: {e}", + dest.display() + )) + })?; + } + // `Continuous` is the UStar 'high-performance' type, specified to + // be treated as a regular file. `GNUSparse` matters in practice: + // redb pre-allocates its files, so GNU tar and rust-tar both store + // the primary/shard DBs as sparse entries, and the tar reader + // transparently expands them here (holes read as zeros) — the + // landed file is the fully-materialized database. + EntryType::Regular | EntryType::Continuous | EntryType::GNUSparse => { + if let Some(dir) = dest.parent() { + std::fs::create_dir_all(dir).map_err(|e| { + py_err(format!( + "failed to create directory {} from prebuilt archive: {e}", + dir.display() + )) + })?; + } + // Copy through an 8 MiB buffer: `Entry::unpack` writes with + // std's 8 KB default, which costs millions of write syscalls + // on a multi-GB archive. + let file = File::create(&dest).map_err(|e| { + py_err(format!( + "failed to create {} while extracting prebuilt archive: {e}", + dest.display() + )) + })?; + let mut out = BufWriter::with_capacity(EXTRACT_READ_BUFFER_BYTES, file); + std::io::copy(&mut entry, &mut out).map_err(|e| { + py_err(format!( + "failed to extract {} from prebuilt archive {}: {e}", + rel.display(), + archive.display() + )) + })?; + out.flush().map_err(|e| { + py_err(format!( + "failed to finish writing {} from prebuilt archive: {e}", + dest.display() + )) + })?; + // Propagate the archived mode (best effort: an unreadable mode + // never blocks landing the bytes). + if let Ok(mode) = entry.header().mode() { + let perms = std::fs::Permissions::from_mode(mode & 0o777); + let _ = std::fs::set_permissions(&dest, perms); + } + } + other => { + return Err(py_err(format!( + "prebuilt archive entry {} has unsupported type {other:?}; \ + the archive must contain only directories and regular \ + (including sparse-encoded) redb files", + rel.display() + ))); + } + } + } + + // Locate the primary + shards among the extracted members (they may be + // nested in subdirectories inside the archive). + let mut shards: BTreeMap = BTreeMap::new(); + let mut primaries: Vec = Vec::new(); + scan_extracted_redb(temp_dir, &mut shards, &mut primaries)?; + primaries.sort(); + let primary = primaries + .iter() + .find(|p| p.file_name() == Some(std::ffi::OsStr::new("fullmap.redb"))) + .or_else(|| primaries.first()) + .ok_or_else(|| { + py_err(format!( + "prebuilt archive {} contains no primary .redb database \ + (expected a non-shard member, preferring `fullmap.redb`)", + archive.display() + )) + })? + .clone(); + + extract_report(progress, "validating"); + + // Validate the extracted bundle against the force-build contract BEFORE + // any rename, reusing the exact read-path helpers so the contract cannot + // drift: exact v5 schema (older tags are rejected loudly), a parseable + // build_id, and the META-advertised shard count. + let validation_ctx = |err: PyErr| { + py_err(format!( + "prebuilt archive {} failed validation: {err}", + archive.display() + )) + }; + let database = open_read_only(&primary).map_err(&validation_ctx)?; + validate_schema(&database).map_err(&validation_ctx)?; + let build_id = read_build_id(&database).map_err(&validation_ctx)?; + let shard_count = shard_count_of(&database).map_err(&validation_ctx)?; + drop(database); + + // The archive must contain EXACTLY s0..s{shard_count-1}: a gap or an + // extra shard (e.g. s16 alongside a 16-shard primary) means a torn or + // wrong archive — refuse instead of landing a broken bundle. + let missing: Vec = (0..shard_count) + .filter(|index| !shards.contains_key(index)) + .collect(); + let extra: Vec = shards + .keys() + .copied() + .filter(|index| *index >= shard_count) + .collect(); + if !missing.is_empty() || !extra.is_empty() { + return Err(py_err(format!( + "prebuilt archive {} has an inconsistent shard set: expected s0..s{}, \ + missing {missing:?}, unexpected {extra:?} — re-download the archive or \ + force a local rebuild with 'tablassert build-fullmap'", + archive.display(), + shard_count - 1 + ))); + } + + // Every shard must open read-only and carry the primary's build_id + // (mirrors the bundle-consistency invariant the read path enforces). + for index in 0..shard_count { + let shard = &shards[&index]; + let shard_db = open_read_only(shard).map_err(|e| { + py_err(format!( + "shard s{index} ({}) of prebuilt archive {} is not a readable fullmap DB: {e}", + shard.display(), + archive.display() + )) + })?; + let shard_build_id = read_build_id(&shard_db).map_err(|e| { + py_err(format!( + "shard s{index} ({}) of prebuilt archive {}: {e}", + shard.display(), + archive.display() + )) + })?; + if shard_build_id != build_id { + return Err(py_err(format!( + "shard s{index} of prebuilt archive {} has build_id {shard_build_id} \ + but the primary has {build_id}; the archive is inconsistent", + archive.display() + ))); + } + } + + // Every check passed: rename into place atomically (the temp dir lives on + // the same filesystem as the output). The primary moves first, mirroring + // the build's commit order; rename replaces any existing file atomically. + std::fs::rename(&primary, output).map_err(|e| { + py_err(format!( + "failed to move extracted primary into place at {}: {e}", + output.display() + )) + })?; + for index in 0..shard_count { + let dest = shard_path(output, index); + std::fs::rename(&shards[&index], &dest).map_err(|e| { + py_err(format!( + "failed to move extracted shard s{index} into place at {}: {e}", + dest.display() + )) + })?; + } + + // Guaranteed cleanup on success: the temp dir is now empty. + std::fs::remove_dir_all(temp_dir).map_err(|e| { + py_err(format!( + "extraction succeeded but cleaning up {} failed: {e}", + temp_dir.display() + )) + })?; + Ok(()) +} + +/// Extract a prebuilt fullmap archive (`fullmap.tar.zst`) into `output` plus +/// its sibling shard files. +/// +/// Streams `File -> BufReader -> zstd decoder -> tar entries` (the multi-GB +/// decompressed tar is never materialized on disk) into a fresh dot-prefixed +/// temp dir inside `output`'s directory (same filesystem -> atomic renames), +/// then VALIDATES the extracted bundle against the same contract the read path +/// enforces — exact v5 schema, a parseable `build_id`, the META-advertised +/// shard count with no gaps or extras, and a matching `build_id` in every +/// shard — BEFORE renaming anything into place: primary -> `output`, shard +/// `i` -> `.s.redb`. A failing validation therefore never +/// lands a broken DB at the output path, and the temp dir is removed on every +/// exit path (guaranteed on success, best-effort on error, so a failed +/// extraction never leaks multi-GB partials). Runs with the GIL released; +/// the optional `progress` callback receives `(detail: str)` updates +/// (`"opening archive"`, `"extracting "`, `"validating"`) and is +/// best-effort. +#[pyfunction] +#[pyo3(signature = (archive, output, progress=None))] +pub fn extract_prebuilt_fullmap( + py: Python<'_>, + archive: PathBuf, + output: PathBuf, + progress: Option>, +) -> PyResult<()> { + // Release the GIL for the whole extract->validate->rename so the multi-GB + // streaming extraction never blocks the interpreter; progress callbacks + // re-acquire it briefly (see `extract_report`). + py.detach(|| extract_prebuilt_fullmap_inner(archive, output, progress)) +} + +fn extract_prebuilt_fullmap_inner( + archive: PathBuf, + output: PathBuf, + progress: Option>, +) -> PyResult<()> { + extract_report(&progress, "opening archive"); + + // A missing or non-file archive errors BEFORE any extraction work. + let meta = std::fs::metadata(&archive).map_err(|e| { + py_err(format!( + "prebuilt archive {} not found: {e}", + archive.display() + )) + })?; + if !meta.is_file() { + return Err(py_err(format!( + "prebuilt archive {} is not a regular file", + archive.display() + ))); + } + + let parent = output + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")); + std::fs::create_dir_all(&parent).map_err(|e| { + py_err(format!( + "failed to create output directory {}: {e}", + parent.display() + )) + })?; + + // Extract into a fresh dot-prefixed temp dir INSIDE the output directory: + // the same filesystem, so the final primary/shard renames are atomic. A + // stale dir from a crashed prior run is removed first, and the dir never + // leaks — cleanup is guaranteed on success and best-effort on every error + // path (a failed extraction must not leave multi-GB partials behind). + let stem = output + .file_stem() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| "fullmap".to_string()); + let temp_dir = parent.join(format!(".{stem}.prebuilt-extract.d")); + if temp_dir.exists() { + std::fs::remove_dir_all(&temp_dir).map_err(|e| { + py_err(format!( + "failed to remove stale extraction dir {}: {e}", + temp_dir.display() + )) + })?; + } + std::fs::create_dir_all(&temp_dir).map_err(|e| { + py_err(format!( + "failed to create extraction dir {}: {e}", + temp_dir.display() + )) + })?; + + match extract_validate_rename(&archive, &temp_dir, &output, &progress) { + Ok(()) => Ok(()), + Err(err) => { + // Best-effort: never leak multi-GB partials from a failed run. + let _ = std::fs::remove_dir_all(&temp_dir); + Err(err) + } + } +} + /// How `open_cached_path` served a handle: as the generation currently living /// at the path, or as the old snapshot cached for a path that is absent. #[derive(Clone, Copy, Debug, Eq, PartialEq)] diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 6247e60..b599994 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -31,13 +31,14 @@ fn xxh64(data: &str) -> String { // build/read path. The `fullmap` module itself stays private; only these // intended entry points are surfaced at the crate root. pub use fullmap::{ - build_fullmap_db, fullmap_source_version, hydrate_categories, hydrate_curies, hydrate_prefixes, - hydrate_sources, lookup_fullmap_terms, + build_fullmap_db, extract_prebuilt_fullmap, fullmap_source_version, hydrate_categories, + hydrate_curies, hydrate_prefixes, hydrate_sources, lookup_fullmap_terms, }; #[pymodule] fn rs(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(fullmap::build_fullmap_db, module)?)?; + module.add_function(wrap_pyfunction!(fullmap::extract_prebuilt_fullmap, module)?)?; module.add_function(wrap_pyfunction!(fullmap::fullmap_source_version, module)?)?; module.add_function(wrap_pyfunction!(fullmap::hydrate_categories, module)?)?; module.add_function(wrap_pyfunction!(fullmap::hydrate_curies, module)?)?; diff --git a/rust/tests/build_golden.rs b/rust/tests/build_golden.rs index 2050dde..6e5dc3b 100644 --- a/rust/tests/build_golden.rs +++ b/rust/tests/build_golden.rs @@ -21,75 +21,20 @@ //! //! All tests are OFFLINE and use `tempfile` scratch dirs. +mod common; + use flate2::write::GzEncoder; use flate2::Compression; -use redb::{Database, ReadableDatabase, ReadableTable, TableDefinition}; -use serde::Deserialize; -use std::collections::{BTreeMap, HashMap}; -use std::fs::File; +use redb::{Database, ReadableDatabase, ReadableTable}; +use std::collections::HashMap; use std::io::Write; -use std::path::{Path, PathBuf}; - -// redb table definitions — MUST match the names/types in `src/fullmap.rs`. -const RECORDS: TableDefinition = TableDefinition::new("records"); -const PREFIXES: TableDefinition = TableDefinition::new("prefixes"); -const CATEGORIES: TableDefinition = TableDefinition::new("categories"); -const SOURCES: TableDefinition = TableDefinition::new("sources"); -const CURIES: TableDefinition = TableDefinition::new("curies"); -const META: TableDefinition<&str, &str> = TableDefinition::new("meta"); - -const SCHEMA_VERSION: &str = "tablassert.fullmap.v5"; -const SHARD_COUNT: usize = 16; - -/// bincode layout MUST match `CurieRow` in `src/fullmap.rs` (field order + types). -#[derive(Deserialize)] -struct CurieRow { - prefix_id: u16, - local_id: String, - preferred_name: String, - category_id: u16, - // Required for the bincode layout; not asserted directly here. - #[allow(dead_code)] - taxon_id: i32, -} - -/// bincode layout MUST match `SourceRow` in `src/fullmap.rs`. -#[derive(Deserialize)] -struct SourceRow { - source_name: String, -} -// --------------------------------------------------------------------------- -// Fixed embedded fixture. Deliberately covers: plain ASCII names, unicode names -// (café / naïve), escaped JSON (quotes, backslashes, \uXXXX), alias fields -// (id/name/categories/taxon), dead terms (12345/none/nan), equivalent -// identifiers, multiple names per row, an empty names array, a null -// preferred_name, a row with no names array, and a class row with no -// equivalent_identifiers. The synonym source file is named "SRC.ndjson" so the -// single source interns as "SRC" (source_id 0). -// --------------------------------------------------------------------------- - -const SYNONYM_LINES: &[&str] = &[ - r#"{"curie":"HGNC:1","preferred_name":"Alpha Gene","names":["Alpha Gene","alpha"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, - r#"{"curie":"HGNC:2","preferred_name":"café","names":["café","naïve"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, - r#"{"curie":"HGNC:3","preferred_name":"Esc","names":["\"Quoted Name\"","back\\slash","\u00e9t\u00e9"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, - r#"{"id":"MONDO:1","name":"Alias Disease","names":["alias disease"],"categories":["biolink:Disease"],"taxon":["NCBITaxon:0"]}"#, - r#"{"curie":"HGNC:4","preferred_name":"Dead","names":["12345","none","nan","realname"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, - r#"{"curie":"HGNC:5","preferred_name":"Empty Names","names":[],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, - r#"{"curie":"HGNC:6","preferred_name":"Shared Hit","names":["shared"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, - r#"{"curie":"MONDO:2","preferred_name":"Shared Disease","names":["shared"],"types":["Disease"],"taxa":["NCBITaxon:0"]}"#, - r#"{"curie":"HGNC:7","preferred_name":null,"names":["nullname"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, - r#"{"curie":"HGNC:8","preferred_name":"No Names","types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, - r#"{"curie":"HGNC:9","preferred_name":"Equiv Free","names":["equivfree"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, - r#"{"curie":"HGNC:10","preferred_name":"Multi A","names":["multi","alpha"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, - r#"{"curie":"MONDO:3","preferred_name":"Multi B","names":["multi"],"types":["Disease"],"taxa":["NCBITaxon:0"]}"#, -]; - -const CLASS_LINES: &[&str] = &[ - r#"{"id":"HGNC:1","equivalent_identifiers":[{"identifier":"NCBIGene:100"},{"identifier":"NCBIGene:101"}]}"#, - r#"{"id":"MONDO:1","equivalent_identifiers":[{"identifier":"DOID:999"}]}"#, - r#"{"id":"HGNC:9"}"#, -]; +// Shared fixture + canonical-form helpers (see `tests/common/mod.rs`). +use common::{ + build_fixture, canonical_dump, open_primary_copy, shard_path, term_curie_map, write_jsonl, + CurieRow, SourceRow, CATEGORIES, CLASS_LINES, CURIES, META, PREFIXES, SCHEMA_VERSION, + SHARD_COUNT, SOURCES, SYNONYM_LINES, +}; /// The pinned canonical output: `term|curie,curie` lines, sorted by term, CURIEs /// sorted within each term. Regenerate with: @@ -148,127 +93,6 @@ t|HGNC:3 été|HGNC:3 "#; -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -fn write_jsonl(path: &Path, lines: &[&str]) { - let mut file = File::create(path).unwrap(); - for line in lines { - writeln!(file, "{line}").unwrap(); - } -} - -/// Sibling shard path for a primary, mirroring `shard_path` in `src/fullmap.rs`. -fn shard_path(primary: &Path, index: usize) -> PathBuf { - let stem = primary.file_stem().unwrap().to_string_lossy().into_owned(); - let ext = primary.extension().unwrap().to_string_lossy().into_owned(); - primary.with_file_name(format!("{stem}.s{index}.{ext}")) -} - -/// Build the fixed fixture at `/fullmap.redb` with `threads` workers and -/// return the primary path. Uses the public `build_fullmap_db` (the production -/// entry point), exactly as Python callers do. -fn build_fixture(dir: &Path, threads: usize) -> PathBuf { - pyo3::Python::initialize(); - let classes = dir.join("classes.ndjson"); - let synonyms = dir.join("SRC.ndjson"); - write_jsonl(&classes, CLASS_LINES); - write_jsonl(&synonyms, SYNONYM_LINES); - let output = dir.join("fullmap.redb"); - pyo3::Python::attach(|py| { - tablassert_rs::build_fullmap_db( - py, - output.clone(), - vec![classes], - vec![synonyms], - Some(threads), - None, - ) - .unwrap(); - }); - output -} - -/// Open a COPY of the (flock-locked) primary so its dims/CURIES/META tables can -/// be read directly. The build commits everything before caching the original, -/// so the copied bytes are a complete, consistent database on a fresh inode. -fn open_primary_copy(primary: &Path) -> Database { - let copy = primary.with_file_name("primary_copy.redb"); - std::fs::copy(primary, ©).unwrap(); - Database::open(©).unwrap() -} - -/// Build the canonical `term -> sorted(CURIE strings)` map by iterating EVERY -/// record across ALL shard files and hydrating curie_ids through the primary's -/// CURIES + PREFIXES tables. Independent of curie_id assignment and thread count. -fn term_curie_map(primary: &Path) -> BTreeMap> { - // prefix_id -> prefix string, and curie_id -> "prefix:local_id". - let db = open_primary_copy(primary); - let read = db.begin_read().unwrap(); - let prefixes = read.open_table(PREFIXES).unwrap(); - let mut prefix_by_id: HashMap = HashMap::new(); - for item in prefixes.iter().unwrap() { - let (id, value) = item.unwrap(); - prefix_by_id.insert(id.value(), value.value().to_string()); - } - drop(prefixes); - let curies = read.open_table(CURIES).unwrap(); - let mut curie_by_id: HashMap = HashMap::new(); - for item in curies.iter().unwrap() { - let (id, bytes) = item.unwrap(); - let row: CurieRow = bincode::deserialize(bytes.value()).unwrap(); - let prefix = prefix_by_id[&row.prefix_id].clone(); - curie_by_id.insert(id.value(), format!("{}:{}", prefix, row.local_id)); - } - drop(curies); - let meta = read.open_table(META).unwrap(); - let shard_count = meta - .get("shards") - .unwrap() - .unwrap() - .value() - .parse::() - .unwrap(); - drop(meta); - drop(read); - drop(db); - - // Iterate every shard's RECORDS, mapping pairs to CURIE strings. - let mut map: BTreeMap> = BTreeMap::new(); - for index in 0..shard_count { - let shard_db = Database::open(shard_path(primary, index)).unwrap(); - let shard_read = shard_db.begin_read().unwrap(); - let records = shard_read.open_table(RECORDS).unwrap(); - for item in records.iter().unwrap() { - let (_hash, bytes) = item.unwrap(); - let (term, pairs): (String, Vec<(u32, u8)>) = - bincode::deserialize(bytes.value()).unwrap(); - let entry = map.entry(term).or_default(); - for (curie_id, _source_id) in pairs { - entry.push(curie_by_id[&curie_id].clone()); - } - } - } - for curie_list in map.values_mut() { - curie_list.sort(); - curie_list.dedup(); - } - map -} - -/// Serialize a `term -> sorted(CURIEs)` map to the canonical multi-line string. -fn canonical_dump(map: &BTreeMap>) -> String { - let mut out = String::new(); - for (term, curie_list) in map { - out.push_str(term); - out.push('|'); - out.push_str(&curie_list.join(",")); - out.push('\n'); - } - out -} - // --------------------------------------------------------------------------- // (a) GOLDEN FILE TEST — the critical pin. // --------------------------------------------------------------------------- diff --git a/rust/tests/common/mod.rs b/rust/tests/common/mod.rs new file mode 100644 index 0000000..5b0785f --- /dev/null +++ b/rust/tests/common/mod.rs @@ -0,0 +1,210 @@ +//! Shared fixture + canonical-form helpers for the fullmap integration tests. +//! +//! Both `build_golden.rs` and `extract_prebuilt.rs` include this module via +//! `mod common;` — the Cargo convention of a `tests/common/mod.rs` keeps it +//! from being compiled as its own test target. The helper BODIES must stay +//! stable: the golden tests pin byte-exact output through `canonical_dump`, +//! and `extract_prebuilt` compares against the exact same canonical form. + +use redb::{Database, ReadableDatabase, ReadableTable, TableDefinition}; +use serde::Deserialize; +use std::collections::{BTreeMap, HashMap}; +use std::fs::File; +use std::io::Write; +use std::path::{Path, PathBuf}; + +// redb table definitions — MUST match the names/types in `src/fullmap.rs`. +pub const RECORDS: TableDefinition = TableDefinition::new("records"); +pub const PREFIXES: TableDefinition = TableDefinition::new("prefixes"); +// CATEGORIES/SOURCES (and `SourceRow` below) are only exercised by +// `build_golden`'s dimension round-trip test today; the targeted allows keep +// this shared module compiling under the crate's `deny(dead_code)` for test +// targets (like `extract_prebuilt`) that do not read those tables. +#[allow(dead_code)] +pub const CATEGORIES: TableDefinition = TableDefinition::new("categories"); +#[allow(dead_code)] +pub const SOURCES: TableDefinition = TableDefinition::new("sources"); +pub const CURIES: TableDefinition = TableDefinition::new("curies"); +pub const META: TableDefinition<&str, &str> = TableDefinition::new("meta"); + +// Only asserted by `build_golden`'s schema pin; the allow keeps this shared +// module compiling under `deny(dead_code)` for targets that never read it. +#[allow(dead_code)] +pub const SCHEMA_VERSION: &str = "tablassert.fullmap.v5"; +pub const SHARD_COUNT: usize = 16; + +/// bincode layout MUST match `CurieRow` in `src/fullmap.rs` (field order + types). +#[derive(Deserialize)] +pub struct CurieRow { + pub prefix_id: u16, + pub local_id: String, + // Required for the bincode layout; only read by some test targets, so the + // allows keep this shared module compiling under `deny(dead_code)` for the + // targets that never read them. + #[allow(dead_code)] + pub preferred_name: String, + #[allow(dead_code)] + pub category_id: u16, + #[allow(dead_code)] + pub taxon_id: i32, +} + +/// bincode layout MUST match `SourceRow` in `src/fullmap.rs`. +#[allow(dead_code)] +#[derive(Deserialize)] +pub struct SourceRow { + pub source_name: String, +} + +// --------------------------------------------------------------------------- +// Fixed embedded fixture. Deliberately covers: plain ASCII names, unicode names +// (café / naïve), escaped JSON (quotes, backslashes, \uXXXX), alias fields +// (id/name/categories/taxon), dead terms (12345/none/nan), equivalent +// identifiers, multiple names per row, an empty names array, a null +// preferred_name, a row with no names array, and a class row with no +// equivalent_identifiers. The synonym source file is named "SRC.ndjson" so the +// single source interns as "SRC" (source_id 0). +// --------------------------------------------------------------------------- + +pub const SYNONYM_LINES: &[&str] = &[ + r#"{"curie":"HGNC:1","preferred_name":"Alpha Gene","names":["Alpha Gene","alpha"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, + r#"{"curie":"HGNC:2","preferred_name":"café","names":["café","naïve"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, + r#"{"curie":"HGNC:3","preferred_name":"Esc","names":["\"Quoted Name\"","back\\slash","\u00e9t\u00e9"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, + r#"{"id":"MONDO:1","name":"Alias Disease","names":["alias disease"],"categories":["biolink:Disease"],"taxon":["NCBITaxon:0"]}"#, + r#"{"curie":"HGNC:4","preferred_name":"Dead","names":["12345","none","nan","realname"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, + r#"{"curie":"HGNC:5","preferred_name":"Empty Names","names":[],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, + r#"{"curie":"HGNC:6","preferred_name":"Shared Hit","names":["shared"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, + r#"{"curie":"MONDO:2","preferred_name":"Shared Disease","names":["shared"],"types":["Disease"],"taxa":["NCBITaxon:0"]}"#, + r#"{"curie":"HGNC:7","preferred_name":null,"names":["nullname"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, + r#"{"curie":"HGNC:8","preferred_name":"No Names","types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, + r#"{"curie":"HGNC:9","preferred_name":"Equiv Free","names":["equivfree"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, + r#"{"curie":"HGNC:10","preferred_name":"Multi A","names":["multi","alpha"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, + r#"{"curie":"MONDO:3","preferred_name":"Multi B","names":["multi"],"types":["Disease"],"taxa":["NCBITaxon:0"]}"#, +]; + +pub const CLASS_LINES: &[&str] = &[ + r#"{"id":"HGNC:1","equivalent_identifiers":[{"identifier":"NCBIGene:100"},{"identifier":"NCBIGene:101"}]}"#, + r#"{"id":"MONDO:1","equivalent_identifiers":[{"identifier":"DOID:999"}]}"#, + r#"{"id":"HGNC:9"}"#, +]; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +pub fn write_jsonl(path: &Path, lines: &[&str]) { + let mut file = File::create(path).unwrap(); + for line in lines { + writeln!(file, "{line}").unwrap(); + } +} + +/// Sibling shard path for a primary, mirroring `shard_path` in `src/fullmap.rs`. +pub fn shard_path(primary: &Path, index: usize) -> PathBuf { + let stem = primary.file_stem().unwrap().to_string_lossy().into_owned(); + let ext = primary.extension().unwrap().to_string_lossy().into_owned(); + primary.with_file_name(format!("{stem}.s{index}.{ext}")) +} + +/// Build the fixed fixture at `/fullmap.redb` with `threads` workers and +/// return the primary path. Uses the public `build_fullmap_db` (the production +/// entry point), exactly as Python callers do. +pub fn build_fixture(dir: &Path, threads: usize) -> PathBuf { + pyo3::Python::initialize(); + let classes = dir.join("classes.ndjson"); + let synonyms = dir.join("SRC.ndjson"); + write_jsonl(&classes, CLASS_LINES); + write_jsonl(&synonyms, SYNONYM_LINES); + let output = dir.join("fullmap.redb"); + pyo3::Python::attach(|py| { + tablassert_rs::build_fullmap_db( + py, + output.clone(), + vec![classes], + vec![synonyms], + Some(threads), + None, + ) + .unwrap(); + }); + output +} + +/// Open a COPY of the (flock-locked) primary so its dims/CURIES/META tables can +/// be read directly. The build commits everything before caching the original, +/// so the copied bytes are a complete, consistent database on a fresh inode. +pub fn open_primary_copy(primary: &Path) -> Database { + let copy = primary.with_file_name("primary_copy.redb"); + std::fs::copy(primary, ©).unwrap(); + Database::open(©).unwrap() +} + +/// Build the canonical `term -> sorted(CURIE strings)` map by iterating EVERY +/// record across ALL shard files and hydrating curie_ids through the primary's +/// CURIES + PREFIXES tables. Independent of curie_id assignment and thread count. +pub fn term_curie_map(primary: &Path) -> BTreeMap> { + // prefix_id -> prefix string, and curie_id -> "prefix:local_id". + let db = open_primary_copy(primary); + let read = db.begin_read().unwrap(); + let prefixes = read.open_table(PREFIXES).unwrap(); + let mut prefix_by_id: HashMap = HashMap::new(); + for item in prefixes.iter().unwrap() { + let (id, value) = item.unwrap(); + prefix_by_id.insert(id.value(), value.value().to_string()); + } + drop(prefixes); + let curies = read.open_table(CURIES).unwrap(); + let mut curie_by_id: HashMap = HashMap::new(); + for item in curies.iter().unwrap() { + let (id, bytes) = item.unwrap(); + let row: CurieRow = bincode::deserialize(bytes.value()).unwrap(); + let prefix = prefix_by_id[&row.prefix_id].clone(); + curie_by_id.insert(id.value(), format!("{}:{}", prefix, row.local_id)); + } + drop(curies); + let meta = read.open_table(META).unwrap(); + let shard_count = meta + .get("shards") + .unwrap() + .unwrap() + .value() + .parse::() + .unwrap(); + drop(meta); + drop(read); + drop(db); + + // Iterate every shard's RECORDS, mapping pairs to CURIE strings. + let mut map: BTreeMap> = BTreeMap::new(); + for index in 0..shard_count { + let shard_db = Database::open(shard_path(primary, index)).unwrap(); + let shard_read = shard_db.begin_read().unwrap(); + let records = shard_read.open_table(RECORDS).unwrap(); + for item in records.iter().unwrap() { + let (_hash, bytes) = item.unwrap(); + let (term, pairs): (String, Vec<(u32, u8)>) = + bincode::deserialize(bytes.value()).unwrap(); + let entry = map.entry(term).or_default(); + for (curie_id, _source_id) in pairs { + entry.push(curie_by_id[&curie_id].clone()); + } + } + } + for curie_list in map.values_mut() { + curie_list.sort(); + curie_list.dedup(); + } + map +} + +/// Serialize a `term -> sorted(CURIEs)` map to the canonical multi-line string. +pub fn canonical_dump(map: &BTreeMap>) -> String { + let mut out = String::new(); + for (term, curie_list) in map { + out.push_str(term); + out.push('|'); + out.push_str(&curie_list.join(",")); + out.push('\n'); + } + out +} diff --git a/rust/tests/extract_prebuilt.rs b/rust/tests/extract_prebuilt.rs new file mode 100644 index 0000000..9a22a1f --- /dev/null +++ b/rust/tests/extract_prebuilt.rs @@ -0,0 +1,123 @@ +//! Positive round-trip integration test for `extract_prebuilt_fullmap`. +//! +//! WHY this test exists: the whole point of distributing a prebuilt +//! `fullmap.tar.zst` is that extracting it must land EXACTLY the database a +//! local force build (`build_fullmap_db`) produces — any divergence silently +//! corrupts every lookup downstream. This test builds the shared fixture DB +//! with the production build entry point, packages the primary + all 16 +//! shards into a zstd-compressed tar (nested in a subdirectory, as real +//! archives may be laid out), extracts it through the PUBLIC +//! `extract_prebuilt_fullmap` pyfunction — including its `py.detach` +//! GIL-release path, driven with the same `Python::initialize` + +//! `Python::attach` pattern `build_golden` uses for `build_fullmap_db` — and +//! pins: +//! * the primary lands at the requested output path; +//! * all 16 shards land as `.s.redb` with no gaps and no extras; +//! * the dot-prefixed temp dir is fully removed on success; +//! * `canonical_dump(term_curie_map(...))` of the extracted bundle EQUALS +//! that of the force-built fixture — the "matches what a force build +//! produces" equivalence pin, in the exact canonical form `build_golden` +//! pins (term-sorted lines, CURIE strings sorted within each term, so it +//! is invariant under thread count and curie_id assignment). +//! +//! Fixtures and canonical-form helpers are shared with `build_golden` via +//! `tests/common/mod.rs` so both tests compare against one definition of the +//! canonical form. Negative paths (torn archives, schema/build_id mismatches, +//! temp-dir cleanup on failure) are covered by a follow-up story. + +mod common; + +use std::fs::File; + +#[test] +fn extract_prebuilt_matches_force_build() { + // 1. Build the fixture with the production entry point (single-threaded + // for a deterministic, fast build). + let fixture_dir = tempfile::tempdir().unwrap(); + let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + + // 2. Package the primary + all 16 shards into `fullmap.tar.zst`. Members + // are nested under a `bundle/` subdirectory to exercise the recursive + // scan path, and an explicit directory entry exercises the + // `EntryType::Directory` extraction branch. + let archive_path = fixture_dir.path().join("fullmap.tar.zst"); + { + let archive_file = File::create(&archive_path).unwrap(); + let encoder = zstd::Encoder::new(archive_file, 0).unwrap(); + let mut builder = tar::Builder::new(encoder); + + let mut dir_header = tar::Header::new_gnu(); + dir_header.set_path("bundle/").unwrap(); + dir_header.set_entry_type(tar::EntryType::Directory); + dir_header.set_size(0); + dir_header.set_mode(0o755); + dir_header.set_cksum(); + builder.append(&dir_header, std::io::empty()).unwrap(); + + let mut primary_file = File::open(&fixture_primary).unwrap(); + builder + .append_file("bundle/fullmap.redb", &mut primary_file) + .unwrap(); + for index in 0..common::SHARD_COUNT { + let shard = common::shard_path(&fixture_primary, index); + let member = format!("bundle/{}", shard.file_name().unwrap().to_string_lossy()); + let mut shard_file = File::open(&shard).unwrap(); + builder.append_file(&member, &mut shard_file).unwrap(); + } + + // Flush tar, then finalize the zstd frame so the file is complete. + let encoder = builder.into_inner().unwrap(); + encoder.finish().unwrap(); + } + + // 3. Extract through the public pyfunction, exactly as Python callers do + // (the GIL token is required by the signature; `Python::attach` + + // `py.detach` inside the function is the production code path). + let out_dir = tempfile::tempdir().unwrap(); + let output = out_dir.path().join("fullmap.redb"); + pyo3::Python::initialize(); + pyo3::Python::attach(|py| { + tablassert_rs::extract_prebuilt_fullmap(py, archive_path.clone(), output.clone(), None) + .unwrap(); + }); + + // 4. The primary landed at `output` and exactly s0..s15 exist beside it. + assert!(output.exists(), "primary must land at the output path"); + for index in 0..common::SHARD_COUNT { + let shard = common::shard_path(&output, index); + assert!(shard.exists(), "missing extracted shard s{index}"); + } + assert!( + !common::shard_path(&output, common::SHARD_COUNT).exists(), + "s16 must not exist" + ); + + // 5. The temp dir is removed on success: no `.fullmap.prebuilt-extract.d` + // (and no dot-prefixed stray at all) may remain in the output dir. + assert!( + !out_dir.path().join(".fullmap.prebuilt-extract.d").exists(), + "temp dir must be removed after a successful extraction" + ); + let strays: Vec = std::fs::read_dir(out_dir.path()) + .unwrap() + .map(|item| item.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| name.starts_with('.')) + .collect(); + assert!( + strays.is_empty(), + "no dot-prefixed files may remain in the output dir, found: {strays:?}" + ); + + // 6. Equivalence pin: the extracted bundle must be canonically identical + // to the force-built fixture. + let built = common::canonical_dump(&common::term_curie_map(&fixture_primary)); + let extracted = common::canonical_dump(&common::term_curie_map(&output)); + assert!( + !built.is_empty(), + "fixture must produce at least one indexed term" + ); + assert_eq!( + extracted, built, + "extracted prebuilt archive must match the force build exactly" + ); +} diff --git a/src/tablassert/rs.pyi b/src/tablassert/rs.pyi index ef6c4fd..863abf6 100644 --- a/src/tablassert/rs.pyi +++ b/src/tablassert/rs.pyi @@ -8,6 +8,7 @@ def build_fullmap_db( output: Path, classes: list[Path], synonyms: list[Path], threads: int | None = None, progress: Callable[[int, int, int, str], None] | None = None ) -> None: ... def dedup_ndjson(input: Path, output: Path, is_edges: bool, domain: str | None = None) -> None: ... +def extract_prebuilt_fullmap(archive: Path, output: Path, progress: Callable[[str], None] | None = None) -> None: ... def fullmap_source_version() -> str: ... def hydrate_categories(db: Path) -> list[str]: ... def hydrate_curies(db: Path, curie_ids: list[int]) -> list[dict[str, Any]]: ... From 0c07dd0b26122b40a45301977eac8bf7d6a5e186 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Fri, 14 Aug 2026 10:30:05 -0700 Subject: [PATCH 2/6] test(rust): loud-failure negatives for prebuilt extraction + PAX-sparse/multi-primary/zstd-magic hardening [US-002] --- rust/src/fullmap.rs | 80 +++++- rust/tests/extract_prebuilt.rs | 460 ++++++++++++++++++++++++++++++++- 2 files changed, 536 insertions(+), 4 deletions(-) diff --git a/rust/src/fullmap.rs b/rust/src/fullmap.rs index 8d14abe..b6c15e8 100644 --- a/rust/src/fullmap.rs +++ b/rust/src/fullmap.rs @@ -2592,7 +2592,27 @@ fn extract_validate_rename( archive.display() )) })?; - let reader = BufReader::with_capacity(EXTRACT_READ_BUFFER_BYTES, file); + let mut reader = BufReader::with_capacity(EXTRACT_READ_BUFFER_BYTES, file); + // Cheap upfront sanity: a zstd stream starts with the 4-byte frame magic + // 0xFD2FB528 (little-endian). `zstd::Decoder::new` is LAZY — it only + // fails on the first read — so garbage bytes from a torn download would + // otherwise surface as a cryptic mid-stream tar error instead of the + // actionable "not valid zstd". + const ZSTD_FRAME_MAGIC: [u8; 4] = [0x28, 0xB5, 0x2F, 0xFD]; + let head = reader.fill_buf().map_err(|e| { + py_err(format!( + "failed to read prebuilt archive {}: {e}", + archive.display() + )) + })?; + if head.len() < ZSTD_FRAME_MAGIC.len() || !head.starts_with(&ZSTD_FRAME_MAGIC) { + return Err(py_err(format!( + "prebuilt archive {} is not valid zstd (missing the zstd frame magic); \ + re-download the archive or force a local rebuild with \ + 'tablassert build-fullmap'", + archive.display() + ))); + } let decoder = zstd::Decoder::new(reader).map_err(|e| { py_err(format!( "prebuilt archive {} is not valid zstd: {e}", @@ -2615,6 +2635,38 @@ fn extract_validate_rename( )) })?; let rel = validated_entry_path(&entry)?; + // tar-rs silently IGNORES `GNU.sparse.*` PAX records (bsdtar's sparse + // encoding), so a PAX-sparse archive of redb files — which are + // genuinely sparse because redb preallocates — would extract WRONG + // bytes with no error. Reject such archives loudly instead of landing + // a silently-corrupt database. (GNU tar's `GNUSparse` entry type does + // NOT use these PAX records and is expanded correctly below.) + if let Some(extensions) = entry.pax_extensions().map_err(|e| { + py_err(format!( + "failed to read the PAX extensions of entry {} in prebuilt archive {}: {e}", + rel.display(), + archive.display() + )) + })? { + for extension in extensions { + let extension = extension.map_err(|e| { + py_err(format!( + "failed to parse a PAX extension of entry {} in prebuilt archive {}: {e}", + rel.display(), + archive.display() + )) + })?; + if extension.key_bytes().starts_with(b"GNU.sparse") { + return Err(py_err(format!( + "prebuilt archive entry {} uses PAX sparse records ({}), which are \ + unsupported: extracting them would silently land wrong bytes for the \ + sparse file — repack with GNU tar or re-publish the archive", + rel.display(), + String::from_utf8_lossy(extension.key_bytes()) + ))); + } + } + } extract_report(progress, &format!("extracting {}", rel.display())); let dest = temp_dir.join(&rel); match entry.header().entry_type() { @@ -2688,6 +2740,32 @@ fn extract_validate_rename( let mut primaries: Vec = Vec::new(); scan_extracted_redb(temp_dir, &mut shards, &mut primaries)?; primaries.sort(); + // More than one non-shard .redb means a torn or mispackaged archive: + // proceeding would silently discard every unchosen candidate, so refuse + // loudly and list them. The ONE documented exception is the prefer-case + // itself — exactly one candidate literally named `fullmap.redb` wins over + // any strays (preserving the historical selection rule). + if primaries.len() > 1 { + let named_fullmap = primaries + .iter() + .filter(|p| p.file_name() == Some(std::ffi::OsStr::new("fullmap.redb"))) + .count(); + if named_fullmap != 1 { + let listing = primaries + .iter() + .map(|p| p.strip_prefix(temp_dir).unwrap_or(p).display().to_string()) + .collect::>() + .join(", "); + return Err(py_err(format!( + "prebuilt archive {} contains {} primary .redb candidates ({listing}); exactly \ + one non-shard .redb is required — a stray primary means a torn or mispackaged \ + archive, so re-download it or force a local rebuild with \ + 'tablassert build-fullmap'", + archive.display(), + primaries.len() + ))); + } + } let primary = primaries .iter() .find(|p| p.file_name() == Some(std::ffi::OsStr::new("fullmap.redb"))) diff --git a/rust/tests/extract_prebuilt.rs b/rust/tests/extract_prebuilt.rs index 9a22a1f..68b286a 100644 --- a/rust/tests/extract_prebuilt.rs +++ b/rust/tests/extract_prebuilt.rs @@ -1,4 +1,6 @@ -//! Positive round-trip integration test for `extract_prebuilt_fullmap`. +//! Integration tests for `extract_prebuilt_fullmap`. +//! +//! ## Positive round-trip //! //! WHY this test exists: the whole point of distributing a prebuilt //! `fullmap.tar.zst` is that extracting it must land EXACTLY the database a @@ -20,14 +22,128 @@ //! pins (term-sorted lines, CURIE strings sorted within each term, so it //! is invariant under thread count and curie_id assignment). //! +//! ## Negative / edge coverage (US-002) +//! +//! WHY: a prebuilt archive arrives over the network and may be torn, corrupt, +//! outdated, or outright hostile. The extraction contract is that a failure +//! must (a) raise an error carrying ACTIONABLE context (what is wrong and how +//! to recover), (b) leave NO primary/shards beside `output` (validation +//! precedes every rename), and (c) leave NO `..prebuilt-extract.d` +//! residue (cleanup runs on every error path). Every negative test pins all +//! three, because a "failure" that still lands a broken DB or leaks multi-GB +//! partials is worse than no extraction at all. +//! //! Fixtures and canonical-form helpers are shared with `build_golden` via //! `tests/common/mod.rs` so both tests compare against one definition of the -//! canonical form. Negative paths (torn archives, schema/build_id mismatches, -//! temp-dir cleanup on failure) are covered by a follow-up story. +//! canonical form. mod common; use std::fs::File; +use std::path::{Path, PathBuf}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/// Create a zstd-compressed tar at `archive_path`; `populate` appends the +/// members (plain `append_file` for fixture files, manual headers for hostile +/// entries). Flushes the tar and finalizes the zstd frame so the archive is +/// complete on disk. +fn write_tar_zst_with( + archive_path: &Path, + populate: impl FnOnce(&mut tar::Builder>), +) { + let archive_file = File::create(archive_path).unwrap(); + let encoder = zstd::Encoder::new(archive_file, 0).unwrap(); + let mut builder = tar::Builder::new(encoder); + populate(&mut builder); + let encoder = builder.into_inner().unwrap(); + encoder.finish().unwrap(); +} + +/// Package `(member name, source file)` pairs into a zstd-compressed tar at +/// `archive_path` (flat layout, no directory entries). +fn package_tar_zst(archive_path: &Path, members: &[(String, PathBuf)]) { + write_tar_zst_with(archive_path, |builder| { + for (name, source) in members { + let mut file = File::open(source).unwrap(); + builder.append_file(name, &mut file).unwrap(); + } + }); +} + +/// `(member name, file path)` pairs for shards `0..count` of +/// `fixture_primary` (member names identical to the on-disk file names). +fn shard_members(fixture_primary: &Path, count: usize) -> Vec<(String, PathBuf)> { + (0..count) + .map(|index| { + let shard = common::shard_path(fixture_primary, index); + ( + shard.file_name().unwrap().to_string_lossy().into_owned(), + shard, + ) + }) + .collect() +} + +/// Run `extract_prebuilt_fullmap` exactly as Python callers do and require it +/// to fail; assert the error message carries every expected context fragment +/// (each negative must tell the user WHAT is wrong and HOW to recover). +fn extract_expect_error(archive: &Path, output: &Path, fragments: &[&str]) { + pyo3::Python::initialize(); + let error = pyo3::Python::attach(|py| { + tablassert_rs::extract_prebuilt_fullmap( + py, + archive.to_path_buf(), + output.to_path_buf(), + None, + ) + .expect_err("extraction of a broken/hostile archive must fail") + .to_string() + }); + for fragment in fragments { + assert!( + error.contains(fragment), + "error {error:?} must carry the actionable context {fragment:?}" + ); + } +} + +/// Assert a failed extraction landed NOTHING: no primary at `output`, no +/// sibling shards, and no dot-prefixed residue (the +/// `..prebuilt-extract.d` temp dir included) in the output dir — a +/// failed extraction must never leave multi-GB partials behind. +fn assert_failed_extraction_left_nothing(out_dir: &Path, output: &Path) { + assert!( + !output.exists(), + "no primary may land at {} after a failed extraction", + output.display() + ); + for index in 0..common::SHARD_COUNT { + let shard = common::shard_path(output, index); + assert!( + !shard.exists(), + "no shard s{index} may land beside the output after a failed extraction" + ); + } + if out_dir.exists() { + let strays: Vec = std::fs::read_dir(out_dir) + .unwrap() + .map(|item| item.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| name.starts_with('.')) + .collect(); + assert!( + strays.is_empty(), + "a failed extraction must leave no dot-prefixed residue in {}, found: {strays:?}", + out_dir.display() + ); + } +} + +// --------------------------------------------------------------------------- +// Positive round-trip (US-001) +// --------------------------------------------------------------------------- #[test] fn extract_prebuilt_matches_force_build() { @@ -121,3 +237,341 @@ fn extract_prebuilt_matches_force_build() { "extracted prebuilt archive must match the force build exactly" ); } + +// --------------------------------------------------------------------------- +// Negatives: corrupt / torn archives +// --------------------------------------------------------------------------- + +/// WHY: a torn download (garbage bytes, not even a zstd frame) must fail at +/// the zstd layer BEFORE any extraction work — and, like every failure, leave +/// no primary, no shards, and no temp-dir residue beside `output`. +#[test] +fn corrupt_archive_is_rejected_and_leaves_nothing() { + let dir = tempfile::tempdir().unwrap(); + let archive = dir.path().join("fullmap.tar.zst"); + std::fs::write(&archive, b"not zstd at all").unwrap(); + let out_dir = tempfile::tempdir().unwrap(); + let output = out_dir.path().join("fullmap.redb"); + extract_expect_error(&archive, &output, &["not valid zstd"]); + assert_failed_extraction_left_nothing(out_dir.path(), &output); +} + +/// WHY: an archive holding ONLY shards (the primary lost in a torn upload) +/// cannot serve a single lookup; the absence must be named explicitly instead +/// of failing later with a cryptic open error. +#[test] +fn archive_without_primary_is_rejected() { + let fixture_dir = tempfile::tempdir().unwrap(); + let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let archive = fixture_dir.path().join("fullmap.tar.zst"); + package_tar_zst( + &archive, + &shard_members(&fixture_primary, common::SHARD_COUNT), + ); + let out_dir = tempfile::tempdir().unwrap(); + let output = out_dir.path().join("fullmap.redb"); + extract_expect_error(&archive, &output, &["no primary"]); + assert_failed_extraction_left_nothing(out_dir.path(), &output); +} + +/// WHY: losing shard s15 mid-transfer is the classic torn-archive shape; the +/// error must name the MISSING shard so operators know what to re-fetch +/// (silently landing 15 shards would make every s15 term unfindable). +#[test] +fn missing_shard_is_rejected_and_named() { + let fixture_dir = tempfile::tempdir().unwrap(); + let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let mut members = vec![("fullmap.redb".to_string(), fixture_primary.clone())]; + members.extend(shard_members(&fixture_primary, common::SHARD_COUNT - 1)); // s0..s14 + let archive = fixture_dir.path().join("fullmap.tar.zst"); + package_tar_zst(&archive, &members); + let out_dir = tempfile::tempdir().unwrap(); + let output = out_dir.path().join("fullmap.redb"); + extract_expect_error( + &archive, + &output, + &["inconsistent shard set", "missing [15]"], + ); + assert_failed_extraction_left_nothing(out_dir.path(), &output); +} + +/// WHY: an s16 beside a 16-shard primary means wrong or torn packaging; the +/// unexpected shard must be named (its contents are never even read — the +/// shard-set check fires first). +#[test] +fn extra_shard_is_rejected_and_named() { + let fixture_dir = tempfile::tempdir().unwrap(); + let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + // No s16 exists in a real build; misuse a copy of s0's bytes as the stray. + let stray = fixture_dir.path().join("fullmap.s16.redb"); + std::fs::copy(common::shard_path(&fixture_primary, 0), &stray).unwrap(); + let mut members = vec![("fullmap.redb".to_string(), fixture_primary.clone())]; + members.extend(shard_members(&fixture_primary, common::SHARD_COUNT)); + members.push(("fullmap.s16.redb".to_string(), stray)); + let archive = fixture_dir.path().join("fullmap.tar.zst"); + package_tar_zst(&archive, &members); + let out_dir = tempfile::tempdir().unwrap(); + let output = out_dir.path().join("fullmap.redb"); + extract_expect_error( + &archive, + &output, + &["inconsistent shard set", "unexpected [16]"], + ); + assert_failed_extraction_left_nothing(out_dir.path(), &output); +} + +/// WHY: a nonexistent archive path (wrong flag, unfetched file) must produce +/// a plain "not found" error BEFORE any output-dir side effects, so a retry +/// with the right path starts clean. +#[test] +fn missing_archive_is_a_not_found_error() { + let dir = tempfile::tempdir().unwrap(); + let archive = dir.path().join("does-not-exist.tar.zst"); + let out_dir = tempfile::tempdir().unwrap(); + let output = out_dir.path().join("fullmap.redb"); + extract_expect_error(&archive, &output, &["not found"]); + assert_failed_extraction_left_nothing(out_dir.path(), &output); +} + +// --------------------------------------------------------------------------- +// Negatives: invalid database content +// --------------------------------------------------------------------------- + +/// Downgrade a COPY of the primary to the v1 schema tag, exactly the shape an +/// outdated published archive would carry. `Database::open` takes a WRITABLE +/// handle on the copy (the `open_primary_copy` precedent, minus the read-only +/// use): overwrite META.schema, commit, drop. The original stays untouched +/// (and flock-locked in the DB cache), hence the copy. +fn outdated_primary_copy(fixture_primary: &Path) -> PathBuf { + let outdated = fixture_primary.with_file_name("outdated.redb"); + std::fs::copy(fixture_primary, &outdated).unwrap(); + let db = redb::Database::open(&outdated).unwrap(); + let txn = db.begin_write().unwrap(); + { + let mut meta = txn.open_table(common::META).unwrap(); + meta.insert("schema", "tablassert.fullmap.v1").unwrap(); + } + txn.commit().unwrap(); + drop(db); + outdated +} + +/// WHY: prebuilt archives outlive schema bumps; extracting a v1..v4 bundle +/// would land a DB that every lookup immediately rejects. The error must +/// carry the wrapped `validate_schema` demand to rebuild, and nothing may +/// land. +#[test] +fn outdated_schema_is_rejected_and_demands_rebuild() { + let fixture_dir = tempfile::tempdir().unwrap(); + let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let outdated = outdated_primary_copy(&fixture_primary); + let mut members = vec![("fullmap.redb".to_string(), outdated)]; + members.extend(shard_members(&fixture_primary, common::SHARD_COUNT)); + let archive = fixture_dir.path().join("fullmap.tar.zst"); + package_tar_zst(&archive, &members); + let out_dir = tempfile::tempdir().unwrap(); + let output = out_dir.path().join("fullmap.redb"); + extract_expect_error( + &archive, + &output, + &["failed validation", "outdated", "build-fullmap"], + ); + assert_failed_extraction_left_nothing(out_dir.path(), &output); +} + +/// WHY: a byte-corrupted (or swapped) primary must fail at `open_read_only` +/// with the validation context — never land — even when all 16 shards are +/// intact and valid. +#[test] +fn non_redb_primary_is_rejected() { + let fixture_dir = tempfile::tempdir().unwrap(); + let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let garbage = fixture_dir.path().join("garbage.redb"); + std::fs::write(&garbage, b"this is definitely not a redb database").unwrap(); + let mut members = vec![("fullmap.redb".to_string(), garbage)]; + members.extend(shard_members(&fixture_primary, common::SHARD_COUNT)); + let archive = fixture_dir.path().join("fullmap.tar.zst"); + package_tar_zst(&archive, &members); + let out_dir = tempfile::tempdir().unwrap(); + let output = out_dir.path().join("fullmap.redb"); + extract_expect_error(&archive, &output, &["failed validation"]); + assert_failed_extraction_left_nothing(out_dir.path(), &output); +} + +// --------------------------------------------------------------------------- +// Negatives: hostile archive entries +// --------------------------------------------------------------------------- + +/// WHY: a hostile `../evil` member must be rejected by the explicit path +/// validation — tar-rs's own `unpack_in` protection SILENTLY SKIPS escaping +/// members, but this contract is a LOUD error — and nothing may be written +/// outside the temp dir: neither beside `output` nor in its parent. +#[test] +fn path_traversal_entry_is_rejected_and_writes_nothing_outside() { + let out_root = tempfile::tempdir().unwrap(); + let archive = out_root.path().join("fullmap.tar.zst"); + write_tar_zst_with(&archive, |builder| { + // tar-rs's `set_path` refuses to WRITE `..` components (write-side + // safety), so smuggle them in by patching the raw name field after a + // benign path — exactly the bytes a hostile archiver would ship. + let mut header = tar::Header::new_gnu(); + header.set_path("evil").unwrap(); + header.set_entry_type(tar::EntryType::Regular); + header.set_size(4); + header.set_mode(0o644); + header.as_mut_bytes()[..7].copy_from_slice(b"../evil"); + header.set_cksum(); + builder.append(&header, &b"evil"[..]).unwrap(); + }); + // Output nested one level deep so BOTH potential escape targets — beside + // `output` (db/evil) and beside its parent (evil) — stay observable. + let out_dir = out_root.path().join("db"); + let output = out_dir.join("fullmap.redb"); + extract_expect_error(&archive, &output, &["unsafe path", "../evil"]); + assert_failed_extraction_left_nothing(&out_dir, &output); + assert!( + !out_dir.join("evil").exists(), + "traversal must not write beside the output" + ); + assert!( + !out_root.path().join("evil").exists(), + "traversal must not escape into the output dir's parent" + ); +} + +/// WHY: a symlink member could point extraction outside the temp dir or fake +/// a `.redb` without real bytes; the contract admits only directories and +/// regular files, so any other entry type errors loudly before any scan. +#[test] +fn symlink_entry_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let archive = dir.path().join("fullmap.tar.zst"); + write_tar_zst_with(&archive, |builder| { + let mut header = tar::Header::new_gnu(); + header.set_path("evil-link.redb").unwrap(); + header.set_entry_type(tar::EntryType::Symlink); + header.set_link_name("fullmap.redb").unwrap(); + header.set_size(0); + header.set_cksum(); + builder.append(&header, std::io::empty()).unwrap(); + }); + let out_dir = tempfile::tempdir().unwrap(); + let output = out_dir.path().join("fullmap.redb"); + extract_expect_error(&archive, &output, &["unsupported type", "Symlink"]); + assert_failed_extraction_left_nothing(out_dir.path(), &output); +} + +/// WHY: tar-rs IGNORES `GNU.sparse.*` PAX records (bsdtar's sparse encoding), +/// and redb files are genuinely sparse (preallocated), so a bsdtar-created +/// PAX archive would silently extract WRONG bytes. The extractor therefore +/// rejects any archive carrying those records; this test builds one the way +/// bsdtar would — a PAX extended header (typeflag 'x') whose records describe +/// the NEXT entry. +#[test] +fn pax_sparse_archive_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let archive = dir.path().join("fullmap.tar.zst"); + write_tar_zst_with(&archive, |builder| { + // PAX record format: " =\n" where counts the + // whole record including the length itself: + // "24 GNU.sparse.size=4096\n" is exactly 24 bytes. + let records = b"24 GNU.sparse.size=4096\n"; + let mut pax_header = tar::Header::new_gnu(); + pax_header.set_path("PaxHeader/fullmap.redb").unwrap(); + pax_header.set_entry_type(tar::EntryType::XHeader); + pax_header.set_size(records.len() as u64); + pax_header.set_mode(0o644); + pax_header.set_cksum(); + builder.append(&pax_header, &records[..]).unwrap(); + + // The regular entry the PAX header describes. + let mut file_header = tar::Header::new_gnu(); + file_header.set_path("fullmap.redb").unwrap(); + file_header.set_entry_type(tar::EntryType::Regular); + file_header.set_size(4); + file_header.set_mode(0o644); + file_header.set_cksum(); + builder.append(&file_header, &b"data"[..]).unwrap(); + }); + let out_dir = tempfile::tempdir().unwrap(); + let output = out_dir.path().join("fullmap.redb"); + extract_expect_error( + &archive, + &output, + &["PAX sparse records", "GNU.sparse.size"], + ); + assert_failed_extraction_left_nothing(out_dir.path(), &output); +} + +// --------------------------------------------------------------------------- +// Negatives + prefer-pin: multiple primaries +// --------------------------------------------------------------------------- + +/// WHY: two non-shard .redb members (neither named exactly `fullmap.redb`) +/// mean torn or mispackaged bytes; before the US-002 hardening the unchosen +/// candidate was silently DISCARDED. Extraction now refuses and lists every +/// candidate by name. +#[test] +fn multiple_unnamed_primaries_are_rejected_and_listed() { + let fixture_dir = tempfile::tempdir().unwrap(); + let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let stray_a = fixture_dir.path().join("primary_a.redb"); + let stray_b = fixture_dir.path().join("primary_b.redb"); + std::fs::copy(&fixture_primary, &stray_a).unwrap(); + std::fs::copy(&fixture_primary, &stray_b).unwrap(); + let mut members = vec![ + ("primary_a.redb".to_string(), stray_a), + ("primary_b.redb".to_string(), stray_b), + ]; + members.extend(shard_members(&fixture_primary, common::SHARD_COUNT)); + let archive = fixture_dir.path().join("fullmap.tar.zst"); + package_tar_zst(&archive, &members); + let out_dir = tempfile::tempdir().unwrap(); + let output = out_dir.path().join("fullmap.redb"); + extract_expect_error( + &archive, + &output, + &[ + "2 primary .redb candidates", + "primary_a.redb", + "primary_b.redb", + ], + ); + assert_failed_extraction_left_nothing(out_dir.path(), &output); +} + +/// WHY: the documented prefer-case must SURVIVE the multi-primary hardening — +/// when exactly one candidate is literally named `fullmap.redb`, it wins over +/// strays (the historical selection rule). Extraction succeeds, the landed +/// bundle matches the force build, and the stray is discarded with the temp +/// dir. +#[test] +fn named_fullmap_primary_is_preferred_over_strays() { + let fixture_dir = tempfile::tempdir().unwrap(); + let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let stray = fixture_dir.path().join("stray.redb"); + std::fs::copy(&fixture_primary, &stray).unwrap(); + let mut members = vec![ + ("fullmap.redb".to_string(), fixture_primary.clone()), + ("stray.redb".to_string(), stray), + ]; + members.extend(shard_members(&fixture_primary, common::SHARD_COUNT)); + let archive = fixture_dir.path().join("fullmap.tar.zst"); + package_tar_zst(&archive, &members); + let out_dir = tempfile::tempdir().unwrap(); + let output = out_dir.path().join("fullmap.redb"); + pyo3::Python::initialize(); + pyo3::Python::attach(|py| { + tablassert_rs::extract_prebuilt_fullmap(py, archive.clone(), output.clone(), None).unwrap(); + }); + assert!( + !out_dir.path().join("stray.redb").exists(), + "the stray primary must be discarded with the temp dir, not landed" + ); + let built = common::canonical_dump(&common::term_curie_map(&fixture_primary)); + let extracted = common::canonical_dump(&common::term_curie_map(&output)); + assert_eq!( + extracted, built, + "the preferred `fullmap.redb` primary must land intact" + ); +} From 42860938a1ec7e9af8d18fec1e31cd056258c514 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Fri, 14 Aug 2026 10:38:24 -0700 Subject: [PATCH 3/6] feat(cli)!: delegate prebuilt fullmap extraction to the Rust extension [US-003] --- src/tablassert/cli.py | 114 ++++----------------- tests/test_cover_cli.py | 220 +++++++++++----------------------------- 2 files changed, 77 insertions(+), 257 deletions(-) diff --git a/src/tablassert/cli.py b/src/tablassert/cli.py index 4dcc706..557511e 100644 --- a/src/tablassert/cli.py +++ b/src/tablassert/cli.py @@ -2,11 +2,8 @@ import hashlib import re -import shutil import subprocess import sys -import tarfile -import tempfile import time from collections.abc import Callable from importlib import import_module @@ -1038,91 +1035,37 @@ def _fetch_prebuilt_sha256(url: str) -> str | None: return None -def _stream_tar(tar: tarfile.TarFile, dest: Path, on_phase: Callable[[str], None]) -> None: - """Extract every member of a streaming tarfile into ``dest`` (data-filtered on 3.12+). +def _extract_prebuilt_fullmap(archive: Path, output: Path, on_phase: Callable[[str], None]) -> None: + """Extract + validate the prebuilt archive via the Rust extension (GIL-free). - Streaming mode (``r|``) only allows extracting each member as it is read (no random - access), which is exactly the loop here. PEP 706 (Python 3.12) added tar-extraction - filters; ``filter="data"`` strips absolute paths, traversals, and unsafe links. On - 3.11 the kwarg is absent and is omitted — the archive is RENCI-published, but the - filter is cheap defense-in-depth when available. - - Args: - tar: An open streaming-mode tarfile. - dest: Directory members are written into. - on_phase: Progress callback fired with the active step label. - """ - use_data_filter: bool = sys.version_info >= (3, 12) - for member in tar: - on_phase(f"extracting {member.name}") - # Explicit branch so pyright sees the "data" literal (PEP 706, Python 3.12+). - if use_data_filter: - tar.extract(member, dest, filter="data") - else: - tar.extract(member, dest) - - -def _extract_zst_tar(archive: Path, dest: Path, on_phase: Callable[[str], None]) -> None: - """Stream-extract a ``.tar.zst`` archive into ``dest`` without materializing the tar on disk. - - Prefers Python 3.14+ native tarfile zstd support; on older runtimes (where ``r|zst`` - raises ``CompressionError``) it streams the archive through the installed ``zstd`` - binary into tarfile. Streaming keeps peak disk near the redb files' own size even - though the uncompressed tar is tens of GB. + Rust streams the ``.tar.zst``, validates it (schema version, build id, exact shard + set), and atomically installs the primary + shards beside ``output`` named after its + stem. Any failure surfaces as ``PrebuiltFullmapUnavailable`` so ``build-fullmap`` can + fall back to a from-scratch BABEL build. Args: archive: Path to the downloaded ``fullmap.tar.zst``. - dest: Directory members are written into (created if missing). + output: Target primary redb path; shards land beside it as ``.s.redb``. on_phase: Progress callback fired with the active step label. Raises: - PrebuiltFullmapUnavailable: If the archive cannot be decompressed/extracted - (native zstd unavailable AND no ``zstd`` binary, a read error, or a zstd failure). + PrebuiltFullmapUnavailable: If decompression, validation, or extraction fails. """ - dest.mkdir(parents=True, exist_ok=True) - on_phase("opening archive") - # Native zstd landed in tarfile for 3.14; older interpreters reject the ``zst`` mode - # with CompressionError, which is caught to fall through to the zstd binary. - try: - with tarfile.open(archive, "r|zst") as tar: - _stream_tar(tar, dest, on_phase) - return - except tarfile.CompressionError: - pass - except (OSError, tarfile.TarError) as exc: - raise PrebuiltFullmapUnavailable(f"failed to read prebuilt archive: {exc}") from exc - - on_phase("streaming via zstd") - binary: str | None = shutil.which("zstd") - if binary is None: - raise PrebuiltFullmapUnavailable("no native zstd support and the `zstd` executable was not found") - try: - proc: subprocess.Popen[bytes] = subprocess.Popen([binary, "-d", "-c", "-T0", str(archive)], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - except OSError as exc: - raise PrebuiltFullmapUnavailable(f"could not start zstd: {exc}") from exc - assert proc.stdout is not None + from tablassert import rs + try: - with tarfile.open(fileobj=proc.stdout, mode="r|") as tar: - _stream_tar(tar, dest, on_phase) - except (OSError, tarfile.TarError) as exc: + rs.extract_prebuilt_fullmap(archive, output, progress=on_phase) + except Exception as exc: raise PrebuiltFullmapUnavailable(f"failed to extract prebuilt archive: {exc}") from exc - finally: - if proc.stdout is not None: - proc.stdout.close() - proc.wait() - if proc.returncode not in (0, None): - stderr: bytes = proc.stderr.read() if proc.stderr else b"" - detail: str = stderr.decode("utf-8", "replace").strip()[-500:] - raise PrebuiltFullmapUnavailable(f"zstd exited with status {proc.returncode}: {detail}") def fetch_prebuilt_fullmap(output: Path, progress: PipelineProgress, version: str = BABEL_VERSION, aria2c: bool = False) -> None: """Download and extract a prebuilt fullmap database from RENCI (instead of building). Two stages: download ``fullmap.tar.zst`` for THIS Tablassert version (cached + - resumable, optionally via ``aria2c``) beside ``output``, then stream-extract it so the - primary redb and its shards land beside ``output`` named after its stem. The checksum - published alongside the archive is verified when present. + resumable, optionally via ``aria2c``) beside ``output``, then extract it in Rust so + the primary redb and its shards land beside ``output`` named after its stem. The + checksum published alongside the archive is verified when present. Args: output: Target primary redb path; the archive is downloaded + extracted beside it. @@ -1175,32 +1118,15 @@ def report_progress(downloaded: int, total: int) -> None: archive.unlink(missing_ok=True) raise PrebuiltFullmapUnavailable(f"checksum mismatch for {archive.name}: expected {expected}, got {actual}") - # Stage 2/2: stream-extract into a temp dir ON THE SAME FILESYSTEM as the output (so - # the renames are atomic), then move the primary + shards beside ``output`` named - # after its stem. A custom --output stem is honored, not assumed to be fullmap.redb. + # Stage 2/2: extract + validate in Rust (streaming zstd+tar, GIL-free). Rust extracts + # to a temp dir on the output's filesystem and atomically renames the primary + shards + # beside ``output`` after its stem — a custom --output stem is honored, not assumed to + # be fullmap.redb. progress.stage("Extracting Fullmap") start, advance, sub_step = progress.section_loop(1, "Extract") start("fullmap.tar.zst") sub_step("extracting") - download_dir.mkdir(parents=True, exist_ok=True) - shard_re: re.Pattern[str] = re.compile(r"\.s(\d+)\.redb$") - with tempfile.TemporaryDirectory(dir=download_dir) as tmp_name: - tmp_dir: Path = Path(tmp_name) - _extract_zst_tar(archive, tmp_dir, on_phase=sub_step) - primary_src: Path | None = None - shards: dict[int, Path] = {} - for candidate in tmp_dir.rglob("*.redb"): - match: re.Match[str] | None = shard_re.search(candidate.name) - if match: - shards[int(match.group(1))] = candidate - elif primary_src is None or candidate.name == "fullmap.redb": - # Prefer a primary literally named fullmap.redb when several non-shard redb files appear. - primary_src = candidate - if primary_src is None: - raise PrebuiltFullmapUnavailable("prebuilt archive contained no primary .redb file") - primary_src.replace(output) - for shard_index, shard_src in sorted(shards.items()): - shard_src.replace(output.parent / f"{output.stem}.s{shard_index}.redb") + _extract_prebuilt_fullmap(archive, output, on_phase=sub_step) # The extracted redb files are the cache; drop the multi-GB archive to free the space. archive.unlink(missing_ok=True) diff --git a/tests/test_cover_cli.py b/tests/test_cover_cli.py index 5ae0984..a7a0615 100644 --- a/tests/test_cover_cli.py +++ b/tests/test_cover_cli.py @@ -738,9 +738,10 @@ def test_fetch_prebuilt_sha256_returns_none_when_no_fullmap_entry(tmp_path: Path def test_fetch_prebuilt_fullmap_downloads_verifies_and_extracts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """End-to-end orchestration: download archive -> verify checksum -> extract beside output. - The download, checksum fetch, and zstd extraction are faked so the test asserts the - ORCHESTRATION: primary + shards land beside ``output`` named after its stem, the verify - branch actually ran against the downloaded bytes, and the archive is removed after extraction. + The download, checksum fetch, and Rust extraction are faked so the test asserts the + ORCHESTRATION: the seam receives the downloaded archive + target output, the files it + installs (primary + shards beside ``output`` named after its stem) survive, the verify + branch actually ran against the downloaded bytes, and the archive is removed afterwards. """ archive_bytes: bytes = b"pretend-archive" digest: str = hashlib.sha256(archive_bytes).hexdigest() @@ -756,17 +757,21 @@ def _fake_download(filename: str, url: str, destination: Path, on_progress: obje monkeypatch.setattr(cli, "download_babel_file", _fake_download) monkeypatch.setattr(cli, "_fetch_prebuilt_sha256", lambda url: digest) - def _fake_extract(archive: Path, dest: Path, on_phase: object) -> None: - dest.mkdir(parents=True, exist_ok=True) - (dest / "fullmap.redb").write_bytes(b"PRIMARY") - (dest / "fullmap.s0.redb").write_bytes(b"SHARD0") - (dest / "fullmap.s1.redb").write_bytes(b"SHARD1") + calls: list[tuple[Path, Path]] = [] - monkeypatch.setattr(cli, "_extract_zst_tar", _fake_extract) + def _fake_extract(archive: Path, output: Path, on_phase: object) -> None: + calls.append((archive, output)) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(b"PRIMARY") + (output.parent / f"{output.stem}.s0.redb").write_bytes(b"SHARD0") + (output.parent / f"{output.stem}.s1.redb").write_bytes(b"SHARD1") + + monkeypatch.setattr(cli, "_extract_prebuilt_fullmap", _fake_extract) output: Path = tmp_path / "data" / "fullmap.redb" cli.fetch_prebuilt_fullmap(output, PipelineProgress(total_stages=2), version="2026jul22") + assert calls == [(output.parent / "fullmap.tar.zst", output)] assert output.read_bytes() == b"PRIMARY" assert (output.parent / "fullmap.s0.redb").read_bytes() == b"SHARD0" assert (output.parent / "fullmap.s1.redb").read_bytes() == b"SHARD1" @@ -782,7 +787,7 @@ def test_fetch_prebuilt_fullmap_checksum_mismatch_unlinks_and_raises(tmp_path: P """ monkeypatch.setattr(cli, "download_babel_file", _write_archive) monkeypatch.setattr(cli, "_fetch_prebuilt_sha256", lambda url: "f" * 64) # never matches - monkeypatch.setattr(cli, "_extract_zst_tar", lambda *a, **k: pytest.fail("extract must not run on a checksum mismatch")) + monkeypatch.setattr(cli, "_extract_prebuilt_fullmap", lambda *a, **k: pytest.fail("extract must not run on a checksum mismatch")) output: Path = tmp_path / "fullmap.redb" with pytest.raises(cli.PrebuiltFullmapUnavailable, match="checksum mismatch"): @@ -796,12 +801,12 @@ def test_fetch_prebuilt_fullmap_missing_checksum_proceeds(tmp_path: Path, monkey monkeypatch.setattr(cli, "_fetch_prebuilt_sha256", lambda url: None) extracted: dict[str, bool] = {"ran": False} - def _fake_extract(archive: Path, dest: Path, on_phase: object) -> None: + def _fake_extract(archive: Path, output: Path, on_phase: object) -> None: extracted["ran"] = True - dest.mkdir(parents=True, exist_ok=True) - (dest / "fullmap.redb").write_bytes(b"PRIMARY") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(b"PRIMARY") - monkeypatch.setattr(cli, "_extract_zst_tar", _fake_extract) + monkeypatch.setattr(cli, "_extract_prebuilt_fullmap", _fake_extract) output: Path = tmp_path / "fullmap.redb" cli.fetch_prebuilt_fullmap(output, PipelineProgress(total_stages=2), version="v") assert extracted["ran"] is True @@ -845,11 +850,11 @@ def _fake_aria2c(filename: str, url: str, destination: Path, retries: int = 5) - monkeypatch.setattr(cli, "download_babel_file_aria2c", _fake_aria2c) monkeypatch.setattr(cli, "_fetch_prebuilt_sha256", lambda url: None) - def _fake_extract(archive: Path, dest: Path, on_phase: object) -> None: - dest.mkdir(parents=True, exist_ok=True) - (dest / "fullmap.redb").write_bytes(b"PRIMARY") + def _fake_extract(archive: Path, output: Path, on_phase: object) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(b"PRIMARY") - monkeypatch.setattr(cli, "_extract_zst_tar", _fake_extract) + monkeypatch.setattr(cli, "_extract_prebuilt_fullmap", _fake_extract) output: Path = tmp_path / "data" / "fullmap.redb" cli.fetch_prebuilt_fullmap(output, PipelineProgress(total_stages=2), version="2026jul22", aria2c=True) @@ -861,175 +866,64 @@ def _fake_extract(archive: Path, dest: Path, on_phase: object) -> None: def test_fetch_prebuilt_fullmap_custom_output_name_renames_shards(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """A custom ``--output`` stem renames the extracted primary + shards to match it. + """A custom ``--output`` stem gets shards named ``.s.redb`` beside it. - WHY: the read path derives shard names from the primary stem (``.s.redb``), so a - prebuilt tarball of ``fullmap.redb`` / ``fullmap.sN.redb`` must be renamed when - ``--output`` is e.g. ``mydb.redb`` or lookups would not find the shards. + WHY: the read path derives shard names from the primary stem (``.s.redb``). The + renaming itself is now ENFORCED BY THE RUST EXTRACTOR (it installs ``.sN.redb`` + directly); this orchestration test fakes the seam and pins that ``fetch_prebuilt_fullmap`` + passes the custom ``output`` through untouched. US-004 covers the real Rust rename e2e. """ monkeypatch.setattr(cli, "download_babel_file", _write_archive) monkeypatch.setattr(cli, "_fetch_prebuilt_sha256", lambda url: None) - def _fake_extract(archive: Path, dest: Path, on_phase: object) -> None: - dest.mkdir(parents=True, exist_ok=True) - (dest / "fullmap.redb").write_bytes(b"PRIMARY") - (dest / "fullmap.s0.redb").write_bytes(b"SHARD0") - (dest / "fullmap.s15.redb").write_bytes(b"SHARD15") + def _fake_extract(archive: Path, output: Path, on_phase: object) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(b"PRIMARY") + (output.parent / f"{output.stem}.s0.redb").write_bytes(b"SHARD0") + (output.parent / f"{output.stem}.s15.redb").write_bytes(b"SHARD15") - monkeypatch.setattr(cli, "_extract_zst_tar", _fake_extract) + monkeypatch.setattr(cli, "_extract_prebuilt_fullmap", _fake_extract) output: Path = tmp_path / "store" / "mydb.redb" cli.fetch_prebuilt_fullmap(output, PipelineProgress(total_stages=2), version="v") assert output.read_bytes() == b"PRIMARY" assert (output.parent / "mydb.s0.redb").read_bytes() == b"SHARD0" assert (output.parent / "mydb.s15.redb").read_bytes() == b"SHARD15" - # the original fullmap.* names did NOT survive the rename - assert not (output.parent / "fullmap.redb").exists() - assert not (output.parent / "fullmap.s0.redb").exists() - - -def test_fetch_prebuilt_fullmap_raises_when_archive_has_no_primary(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """An archive with only shards (no primary .redb) fails loud instead of silently passing.""" - monkeypatch.setattr(cli, "download_babel_file", _write_archive) - monkeypatch.setattr(cli, "_fetch_prebuilt_sha256", lambda url: None) - monkeypatch.setattr( - cli, - "_extract_zst_tar", - lambda archive, dest, on_phase: (dest.mkdir(parents=True, exist_ok=True), (dest / "fullmap.s0.redb").write_bytes(b"SHARD")), - ) - output: Path = tmp_path / "fullmap.redb" - with pytest.raises(cli.PrebuiltFullmapUnavailable, match=r"no primary \.redb"): - cli.fetch_prebuilt_fullmap(output, PipelineProgress(total_stages=2), version="v") -def test_extract_zst_tar_extracts_real_tiny_archive(tmp_path: Path) -> None: - """A real tiny ``.tar.zst`` round-trips through ``_extract_zst_tar`` (native path on 3.14+). +def test_extract_prebuilt_fullmap_seam_wraps_rust_errors(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A Rust extraction failure is wrapped as ``PrebuiltFullmapUnavailable`` (-> build fallback). - WHY: the production archive is a 46 GB ``.tar.zst`` we cannot fetch in tests; this builds a - tiny one and proves the streaming extraction + data filter land members on disk, including - a nested file (parent dirs created automatically). - """ - archive: Path = tmp_path / "fullmap.tar.zst" - _build_tiny_tar_zst(archive, {"fullmap.redb": b"PRIMARY", "fullmap.s0.redb": b"SHARD0", "nested/x.txt": b"hi"}) - dest: Path = tmp_path / "out" - phases: list[str] = [] - cli._extract_zst_tar(archive, dest, on_phase=phases.append) - assert (dest / "fullmap.redb").read_bytes() == b"PRIMARY" - assert (dest / "fullmap.s0.redb").read_bytes() == b"SHARD0" - assert (dest / "nested" / "x.txt").read_bytes() == b"hi" - assert any("extracting" in p for p in phases) - - -def test_extract_zst_tar_falls_back_to_zstd_binary(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """When native zstd is unavailable, the ``zstd`` binary streams the archive into tarfile. - - WHY: Python 3.11-3.13 lack native tarfile zstd, so the binary fallback is the only path - there. On 3.14+ we force the native attempt to raise ``CompressionError`` to exercise the - fallback; skipped when no ``zstd`` binary is installed. + WHY: the Rust extension raises ``RuntimeError`` with actionable context; the seam must + translate it so ``build-fullmap``'s fallback catches one exception type, keeping the + stable "failed to extract prebuilt archive" prefix and the original as ``__cause__``. """ - zstd: str | None = shutil.which("zstd") - if zstd is None: - pytest.skip("no zstd binary to exercise the fallback path") - archive: Path = tmp_path / "fullmap.tar.zst" - _build_tiny_tar_zst(archive, {"fullmap.redb": b"PRIMARY"}) - real_open = cli.tarfile.open + original: RuntimeError = RuntimeError("build_id mismatch: expected abc, got def") - def _force_native_failure(*args: Any, **kwargs: Any) -> Any: - mode: str = args[1] if len(args) > 1 else str(kwargs.get("mode", "")) - if "zst" in mode: - raise cli.tarfile.CompressionError("forced: simulate a pre-3.14 runtime") - return real_open(*args, **kwargs) + def _raiser(archive: Path, output: Path, progress: object = None) -> None: + raise original - monkeypatch.setattr(cli.tarfile, "open", _force_native_failure) - dest: Path = tmp_path / "out" - cli._extract_zst_tar(archive, dest, on_phase=lambda p: None) - assert (dest / "fullmap.redb").read_bytes() == b"PRIMARY" + # The seam imports ``from tablassert import rs`` INSIDE the function, so patch the + # attribute on the ``rs`` module itself. + monkeypatch.setattr(rs, "extract_prebuilt_fullmap", _raiser) + with pytest.raises(cli.PrebuiltFullmapUnavailable, match="failed to extract prebuilt archive") as exc_info: + cli._extract_prebuilt_fullmap(tmp_path / "fullmap.tar.zst", tmp_path / "fullmap.redb", on_phase=lambda p: None) + assert exc_info.value.__cause__ is original -def _force_native_zst_failure(monkeypatch: pytest.MonkeyPatch) -> None: - """Make ``tarfile.open`` reject the ``r|zst`` mode so the zstd-binary path runs on any Python.""" - real_open = cli.tarfile.open +def test_extract_prebuilt_fullmap_seam_does_not_wrap_keyboard_interrupt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """``KeyboardInterrupt`` propagates UNWRAPPED (it is a ``BaseException``, not an error). - def _fake_open(*args: Any, **kwargs: Any) -> Any: - mode: str = args[1] if len(args) > 1 else str(kwargs.get("mode", "")) - if "zst" in mode: - raise cli.tarfile.CompressionError("forced: simulate a pre-3.14 runtime") - return real_open(*args, **kwargs) - - monkeypatch.setattr(cli.tarfile, "open", _fake_open) - - -def test_extract_zst_tar_raises_when_no_zstd_binary(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """No native zstd AND no zstd binary => PrebuiltFullmapUnavailable (-> build fallback). - - WHY: a pre-3.14 Python without the zstd CLI cannot extract the archive, so the default - from-scratch BABEL build is the correct fallback. - """ - archive: Path = tmp_path / "fullmap.tar.zst" - archive.write_bytes(b"not-used") # never reaches the archive before the binary check - _force_native_zst_failure(monkeypatch) - monkeypatch.setattr(cli.shutil, "which", lambda name: None) - with pytest.raises(cli.PrebuiltFullmapUnavailable, match="`zstd` executable was not found"): - cli._extract_zst_tar(archive, tmp_path / "out", on_phase=lambda p: None) - - -def test_extract_zst_tar_raises_when_zstd_binary_fails_to_start(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """A zstd binary present but failing to exec => PrebuiltFullmapUnavailable (defensive OSError).""" - archive: Path = tmp_path / "fullmap.tar.zst" - archive.write_bytes(b"x") - _force_native_zst_failure(monkeypatch) - monkeypatch.setattr(cli.shutil, "which", lambda name: "/usr/bin/zstd") - monkeypatch.setattr(cli.subprocess, "Popen", lambda *a, **k: (_ for _ in ()).throw(OSError("exec failed"))) - with pytest.raises(cli.PrebuiltFullmapUnavailable, match="could not start zstd"): - cli._extract_zst_tar(archive, tmp_path / "out", on_phase=lambda p: None) - - -def test_extract_zst_tar_raises_on_corrupt_archive(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """A corrupt (non-zstd) archive => tarfile read error => PrebuiltFullmapUnavailable (-> build fallback). - - WHY: a truncated/corrupt download must fail loud rather than silently extract garbage. zstd - decompresses nothing, tarfile hits an empty/invalid stream, and the fallback read-error path fires. + WHY: the seam catches ``Exception`` only; wrapping Ctrl-C would misreport a user abort as + a bad archive and trigger the from-scratch build fallback instead of stopping. """ - zstd: str | None = shutil.which("zstd") - if zstd is None: - pytest.skip("no zstd binary to exercise the corrupt-archive path") - archive: Path = tmp_path / "fullmap.tar.zst" - archive.write_bytes(b"this is definitely not a zstd stream") - _force_native_zst_failure(monkeypatch) - with pytest.raises(cli.PrebuiltFullmapUnavailable, match="failed to extract prebuilt archive"): - cli._extract_zst_tar(archive, tmp_path / "out", on_phase=lambda p: None) - -def test_extract_zst_tar_raises_on_nonzero_zstd_exit(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """A non-zero zstd exit AFTER a valid tar stream still fails loud (returncode guard). + def _interrupt(archive: Path, output: Path, progress: object = None) -> None: + raise KeyboardInterrupt - WHY: zstd can write a complete, valid stream and still exit non-zero (e.g. trailing garbage - after the frame). The returncode guard refuses to trust such output. Native is forced off and - a fake Popen serves a real (uncompressed) tar stream while reporting exit 2. - """ - tar_buf: io.BytesIO = io.BytesIO() - with tarfile.open(fileobj=tar_buf, mode="w|") as tar: - info: tarfile.TarInfo = tarfile.TarInfo(name="fullmap.redb") - info.size = 1 - tar.addfile(info, io.BytesIO(b"P")) - tar_buf.seek(0) - - class _FakeProc: - def __init__(self) -> None: - self.stdout = tar_buf - self.stderr = io.BytesIO(b"") - self.returncode = 2 - - def wait(self) -> int: - return 2 - - _force_native_zst_failure(monkeypatch) - monkeypatch.setattr(cli.shutil, "which", lambda name: "/usr/bin/zstd") - monkeypatch.setattr(cli.subprocess, "Popen", lambda *a, **k: _FakeProc()) - archive: Path = tmp_path / "fullmap.tar.zst" - archive.write_bytes(b"ignored") # fake Popen ignores the archive path - with pytest.raises(cli.PrebuiltFullmapUnavailable, match="zstd exited with status 2"): - cli._extract_zst_tar(archive, tmp_path / "out", on_phase=lambda p: None) + monkeypatch.setattr(rs, "extract_prebuilt_fullmap", _interrupt) + with pytest.raises(KeyboardInterrupt): + cli._extract_prebuilt_fullmap(tmp_path / "fullmap.tar.zst", tmp_path / "fullmap.redb", on_phase=lambda p: None) def test_build_fullmap_command_defaults_to_prebuilt_download(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: From a30e51bf3ecb84de73b2013622fa4ce8bb75b5c0 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Fri, 14 Aug 2026 11:02:24 -0700 Subject: [PATCH 4/6] test(cli): real end-to-end prebuilt extraction equals force build [US-004] --- tests/test_cover_cli.py | 200 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) diff --git a/tests/test_cover_cli.py b/tests/test_cover_cli.py index a7a0615..8ece5c2 100644 --- a/tests/test_cover_cli.py +++ b/tests/test_cover_cli.py @@ -9,6 +9,7 @@ from __future__ import annotations +import gzip import hashlib import io import shutil @@ -997,3 +998,202 @@ def parse(argv: list[str]) -> dict[str, Any]: assert parse(["build-fullmap", "-f"])["force"] is True with pytest.raises(UnknownOptionError): parse(["build-fullmap", "--no-force"]) + + +# --- REAL end-to-end prebuilt tests: the Rust extractor runs unmocked (US-004) --- + +# BABEL-format NDJSON lines mirroring the shapes pinned in rust/tests/common/mod.rs +# (CLASS_LINES / SYNONYM_LINES). Two class + two synonym gzip files prove the +# multi-file build path; the names deliberately yield level-one lowercase terms that +# resolve to KNOWN CURIEs (cross-checked against rust/tests/build_golden.rs). +_REAL_CLASS_LINES_HGNC: tuple[str, ...] = ('{"id":"HGNC:1","equivalent_identifiers":[{"identifier":"NCBIGene:100"}]}', '{"id":"HGNC:6"}') +_REAL_CLASS_LINES_MONDO: tuple[str, ...] = ('{"id":"MONDO:2","equivalent_identifiers":[{"identifier":"DOID:999"}]}',) +_REAL_SYNONYM_LINES_HGNC: tuple[str, ...] = ( + '{"curie":"HGNC:1","preferred_name":"Alpha Gene","names":["Alpha Gene","alpha"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}', + '{"curie":"HGNC:2","preferred_name":"café","names":["café","naïve"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}', + '{"curie":"HGNC:6","preferred_name":"Shared Hit","names":["shared"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}', + '{"curie":"HGNC:10","preferred_name":"Multi A","names":["multi","alpha"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}', +) +_REAL_SYNONYM_LINES_MONDO: tuple[str, ...] = ( + '{"curie":"MONDO:2","preferred_name":"Shared Disease","names":["shared"],"types":["Disease"],"taxa":["NCBITaxon:0"]}', +) +# Level-one lowercase forms of the fixture names/preferred names above; every one +# resolves (golden expectations: alpha -> HGNC:1+HGNC:10, alpha gene -> HGNC:1, +# café/naïve -> HGNC:2, shared -> HGNC:6+MONDO:2, multi -> HGNC:10). +_REAL_RESOLVING_TERMS: list[str] = ["alpha", "alpha gene", "café", "naïve", "shared", "multi"] + + +def _write_gzip_ndjson(path: Path, lines: tuple[str, ...]) -> Path: + """Write BABEL-format gzip NDJSON (the on-disk shape ``build_fullmap_db`` downloads).""" + path.parent.mkdir(parents=True, exist_ok=True) + with gzip.open(path, "wt", encoding="utf-8") as handle: + handle.write("\n".join(lines) + "\n") + return path + + +def _build_real_force_fullmap(directory: Path) -> Path: + """Build a REAL fullmap with ``rs.build_fullmap_db`` and return the primary path. + + Two gzip class files + two gzip synonym files exercise the multi-file, gzip-aware + build path exactly as ``build_fullmap_pipeline`` Stage 3 does. + """ + classes: list[Path] = [ + _write_gzip_ndjson(directory / "classes" / "HGNC.ndjson.gz", _REAL_CLASS_LINES_HGNC), + _write_gzip_ndjson(directory / "classes" / "MONDO.ndjson.gz", _REAL_CLASS_LINES_MONDO), + ] + synonyms: list[Path] = [ + _write_gzip_ndjson(directory / "synonyms" / "HGNC.ndjson.gz", _REAL_SYNONYM_LINES_HGNC), + _write_gzip_ndjson(directory / "synonyms" / "MONDO.ndjson.gz", _REAL_SYNONYM_LINES_MONDO), + ] + output: Path = directory / "force" / "fullmap.redb" + rs.build_fullmap_db(output, classes, synonyms, threads=2) + return output + + +def _force_shards(primary: Path) -> list[Path]: + """The 16 shard files ``build_fullmap_db`` lands beside the primary.""" + return [primary.parent / f"{primary.stem}.s{index}.redb" for index in range(16)] + + +def _pack_fullmap_bundle(archive: Path, files: list[Path]) -> None: + """Pack redb files into a real ``tar.zst`` under their basenames (root-level members).""" + _build_tiny_tar_zst(archive, {path.name: path.read_bytes() for path in files}) + + +def _stage_prebuilt_download(monkeypatch: pytest.MonkeyPatch, archive: Path) -> None: + """Point ``fetch_prebuilt_fullmap``'s download seam at a locally staged archive. + + The fake downloader copies the staged bytes into the download dir as the requested + filename (same idiom as ``_write_archive``); the checksum fetch returns ``None`` so + verification is skipped, like the existing missing-checksum orchestration tests. + """ + + def _fake_download(filename: str, url: str, destination: Path, on_progress: object = None) -> Path: + destination.mkdir(parents=True, exist_ok=True) + path: Path = destination / filename + shutil.copyfile(archive, path) + return path + + monkeypatch.setattr(cli, "download_babel_file", _fake_download) + monkeypatch.setattr(cli, "_fetch_prebuilt_sha256", lambda url: None) + + +def _sorted_lookup_rows(db: Path, terms: list[str]) -> list[dict[str, Any]]: + """``lookup_fullmap_terms`` rows sorted by (term, CURIE) so order cannot flake.""" + return sorted(rs.lookup_fullmap_terms(db, terms), key=lambda row: (row["term"], row["CURIE"])) + + +def test_fetch_prebuilt_fullmap_real_archive_matches_force_build(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """THE equivalence contract: an extracted prebuilt equals a force build, for real. + + WHY: users must be able to trust "download prebuilt" as a byte-faithful substitute + for hours of BABEL building. This pins that promise through the REAL Rust pipeline: + ``rs.build_fullmap_db`` builds a genuine 17-file bundle from gzip NDJSON, the bundle + is packed into a real ``tar.zst``, the download is faked (bytes only, no network), + and ``fetch_prebuilt_fullmap`` runs the REAL streaming zstd+tar extractor with ZERO + extraction monkeypatching — then both DBs must answer every lookup identically. + A custom ``--output`` stem (``mymap``) proves stem-naming flows through the Rust + rename path. + """ + force_db: Path = _build_real_force_fullmap(tmp_path / "build") + staging: Path = tmp_path / "staging" + staging.mkdir() + archive: Path = staging / "fullmap.tar.zst" + _pack_fullmap_bundle(archive, [force_db, *_force_shards(force_db)]) + _stage_prebuilt_download(monkeypatch, archive) + + output: Path = tmp_path / "extracted" / "mymap.redb" + cli.fetch_prebuilt_fullmap(output, PipelineProgress(total_stages=2), version="v") + + # Exactly the primary + s0..s15 renamed after the CUSTOM stem — no other redb files. + expected: list[str] = sorted(["mymap.redb", *(f"mymap.s{index}.redb" for index in range(16))]) + landed: list[str] = sorted(path.name for path in output.parent.iterdir() if path.suffix == ".redb") + assert landed == expected + assert not (output.parent / "fullmap.tar.zst").exists() # archive deleted after extraction + assert not (output.parent / ".mymap.prebuilt-extract.d").exists() # no temp-dir residue + + # The extracted DB answers every lookup EXACTLY like the force-built one. + extracted_rows: list[dict[str, Any]] = _sorted_lookup_rows(output, _REAL_RESOLVING_TERMS) + force_rows: list[dict[str, Any]] = _sorted_lookup_rows(force_db, _REAL_RESOLVING_TERMS) + assert extracted_rows # sanity: the fixture terms really do resolve + assert extracted_rows == force_rows + # Spot-pin semantic content so a silent empty-schema DB cannot pass by equality alone. + assert {(row["term"], row["CURIE"]) for row in extracted_rows} >= { + ("alpha", "HGNC:1"), + ("alpha", "HGNC:10"), + ("alpha gene", "HGNC:1"), + ("café", "HGNC:2"), + ("naïve", "HGNC:2"), + ("shared", "HGNC:6"), + ("shared", "MONDO:2"), + ("multi", "HGNC:10"), + } + + +def test_fetch_prebuilt_fullmap_real_corrupt_archive_keeps_archive_and_lands_nothing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Garbage (non-zstd) archive bytes fail loud through the real Rust seam. + + WHY: a torn download must never leave a half-extracted "DB" the read path would + open. The seam wraps the Rust error as ``PrebuiltFullmapUnavailable`` (triggering + the force-build fallback), nothing lands beside the output, and — by design for + extraction failures — the archive is KEPT so the failure is diagnosable. + """ + archive: Path = tmp_path / "staging" / "fullmap.tar.zst" + archive.parent.mkdir() + archive.write_bytes(b"this is definitely not a zstd stream") + _stage_prebuilt_download(monkeypatch, archive) + + output: Path = tmp_path / "extracted" / "mymap.redb" + with pytest.raises(cli.PrebuiltFullmapUnavailable, match="failed to extract prebuilt archive"): + cli.fetch_prebuilt_fullmap(output, PipelineProgress(total_stages=2), version="v") + + assert not output.exists() + assert [path for path in output.parent.iterdir() if path.suffix == ".redb"] == [] + assert not (output.parent / ".mymap.prebuilt-extract.d").exists() + assert (output.parent / "fullmap.tar.zst").exists() # kept on extraction failure + + +def test_fetch_prebuilt_fullmap_real_shards_only_archive_rejects_missing_primary(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """An archive with ONLY the 16 shards (no primary) is rejected before renaming. + + WHY: shard files are useless without the primary (CURIES/PREFIXES/META live there). + A mispackaged archive must fail validation with a named cause, not install an + orphan shard set. + """ + force_db: Path = _build_real_force_fullmap(tmp_path / "build") + staging: Path = tmp_path / "staging" + staging.mkdir() + archive: Path = staging / "fullmap.tar.zst" + _pack_fullmap_bundle(archive, _force_shards(force_db)) # no primary member + _stage_prebuilt_download(monkeypatch, archive) + + output: Path = tmp_path / "extracted" / "mymap.redb" + with pytest.raises(cli.PrebuiltFullmapUnavailable, match="no primary"): + cli.fetch_prebuilt_fullmap(output, PipelineProgress(total_stages=2), version="v") + + assert not output.exists() + assert [path for path in output.parent.iterdir() if path.suffix == ".redb"] == [] + assert (output.parent / "fullmap.tar.zst").exists() # kept on extraction failure + + +def test_fetch_prebuilt_fullmap_real_archive_missing_one_shard_rejected(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Primary + only s0..s14 is an inconsistent shard set and must not land. + + WHY: lookups hashing into the absent shard would fail at read time, silently + corrupting resolution. The Rust validator compares the archive's shard set + against the primary's META-advertised count BEFORE any rename, naming the gap. + """ + force_db: Path = _build_real_force_fullmap(tmp_path / "build") + staging: Path = tmp_path / "staging" + staging.mkdir() + archive: Path = staging / "fullmap.tar.zst" + _pack_fullmap_bundle(archive, [force_db, *_force_shards(force_db)[:15]]) # drop s15 + _stage_prebuilt_download(monkeypatch, archive) + + output: Path = tmp_path / "extracted" / "mymap.redb" + with pytest.raises(cli.PrebuiltFullmapUnavailable, match=r"inconsistent shard set.*missing \[15\]"): + cli.fetch_prebuilt_fullmap(output, PipelineProgress(total_stages=2), version="v") + + assert not output.exists() + assert [path for path in output.parent.iterdir() if path.suffix == ".redb"] == [] + assert (output.parent / "fullmap.tar.zst").exists() # kept on extraction failure From 6350efd75d658cd34edc085b255dd95a2820eda8 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Fri, 14 Aug 2026 11:07:06 -0700 Subject: [PATCH 5/6] docs: prebuilt fullmap extraction is Rust-side and validated against the force-build contract [US-005] --- CHANGELOG.md | 1 + docs/cli.md | 10 +++++++--- docs/fullmap.md | 15 +++++++++++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e50ab7a..e744e47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ All notable changes to this project are documented in this file. **Migration:** add an `effect_type` to any config declaring an effect size (`method: value` when every row shares one statistic, `method: column` when the table provides it); its value is coerced to the permissible `EffectTypes` set as before. Drop a lone `effect_type` that had no effect size — the build was already discarding it. ### Changed +- **`build-fullmap`'s prebuilt extraction now runs in the Rust extension, and extracted archives are validated against the force-build contract before install.** The prebuilt download (and its `sha256sum.txt` check) is unchanged, but the extraction is no longer Python-side (`tarfile` zstd with an installed-`zstd`-binary fallback): the extension streams the archive through zstd → tar — the multi-GB decompressed tar is never materialized on disk — with the GIL released, extracts into a temp directory on the output's filesystem, and validates the bundle BEFORE renaming anything into place: the primary's `meta` schema tag must be exactly `tablassert.fullmap.v5` (older schemas are rejected), a `build_id` must be present, the shard files must be exactly the set the primary advertises — no gaps, no extras — and every shard's `build_id` must equal the primary's. Only a passing bundle is atomically renamed beside `--output` (primary → `--output`, shard `i` → `.s.redb`); unsupported entry types (symlinks, devices) and PAX-sparse archives are rejected loudly, and path traversal is blocked. Any failure raises, so the command falls back to the from-scratch BABEL build exactly as a failed download does (the archive is kept for that retry). For end users the visible effect is faster extraction and the guarantee that an invalid prebuilt can never land a broken database at `--output`; the progress callback keeps firing its step-detail strings (`opening archive`, `extracting `, `validating`). The private Python helpers this replaces — `_extract_zst_tar` / `_stream_tar` — are gone; they were never public API, so the removal breaks only code that reached into those internals. - **The `build-kg --qc` study no longer flags `original_*` fields for leading/trailing whitespace.** Those slots are verbatim copies of the source-table cell (written by `Tcode.encoding` under an `original_` prefix before any regex/normalization runs), so retaining the cell's whitespace is faithful to the source, not a defect. The whitespace assertion now skips any key prefixed `original_`, while every other field is still checked exactly as before. ## 10.1.0 - 2026-08-13 diff --git a/docs/cli.md b/docs/cli.md index c6e75a2..a92580f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -114,9 +114,13 @@ tablassert build-fullmap --aria2c --output /data/fullmap/fullmap.redb By default `build-fullmap` looks for a prebuilt `fullmap.tar.zst` at `https://stars.renci.org/var/babel_outputs//fullmap//` (the version directory is the **installed Tablassert package version**, never hardcoded), verifies it against the -published `sha256sum.txt`, and stream-extracts it beside `--output`. If no prebuilt exists for this -version (or the download/extract fails), it falls back to a from-scratch BABEL build and logs a -warning. A database already present at `--output` is reused as-is; pass `--force` to rebuild. +published `sha256sum.txt`, and extracts it beside `--output` in the Rust extension — streaming zstd → +tar with the GIL released (the decompressed tar never touches disk), then validating the extracted +primary + shards against the force-build contract (exact `v5` schema, a recorded `build_id`, the exact +shard set, and per-shard `build_id` equality) before atomically renaming them into place. If no +prebuilt exists for this version (or the download or extraction fails), it falls back to a +from-scratch BABEL build and logs a warning. A database already present at `--output` is reused as-is; +pass `--force` to rebuild. See [Fullmap](fullmap.md) for the data pipeline, output schema, and graph-config usage. diff --git a/docs/fullmap.md b/docs/fullmap.md index 78354b0..9af9826 100644 --- a/docs/fullmap.md +++ b/docs/fullmap.md @@ -32,10 +32,17 @@ and the `--force` / `-f` rebuild flag), their defaults, and more examples. By default, `build-fullmap` first downloads a **prebuilt** database published for this Tablassert version — a `fullmap.tar.zst` under `.../fullmap//` (the version directory is the installed package version, never hardcoded), verified against a co-published `sha256sum.txt` and -stream-extracted beside `--output`. If none is available for this version it falls back to the -from-scratch build below; `--force` / `-f` skips the prebuilt attempt and always builds. The optional -`--aria2c` / `-a` accelerates **either** download — the multi-GB prebuilt archive is the ideal aria2 -use case. +extracted beside `--output` entirely in the Rust extension: it streams the archive through zstd → tar +(the multi-GB decompressed tar is never materialized on disk) with the GIL released, extracts into a +temp directory on the output's filesystem, and — before renaming anything into place — validates the +bundle against the same contract a `--force` build must satisfy: the `meta` schema tag is exactly +`tablassert.fullmap.v5`, a `build_id` is recorded, the shard files are exactly the set the primary +advertises (no gaps, no extras), and every shard's `build_id` equals the primary's. Only a bundle that +passes is atomically renamed into place (primary → `--output`, shards beside it); any failure raises +and the command falls back to the from-scratch build below. If no prebuilt is published for this +version it falls back the same way; `--force` / `-f` skips the prebuilt attempt and always builds. The +optional `--aria2c` / `-a` accelerates **either** download — the multi-GB prebuilt archive is the ideal +aria2 use case. Two facts matter most when planning a build: From e8387c04eb02d7167e1fb7b6f1c5af5529aa06b9 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Fri, 14 Aug 2026 11:22:14 -0700 Subject: [PATCH 6/6] test(rust): pin progress-callback details + custom-stem rename in extract round-trip [audit] --- rust/src/fullmap.rs | 5 +++ rust/tests/extract_prebuilt.rs | 80 ++++++++++++++++++++++++++++++++-- 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/rust/src/fullmap.rs b/rust/src/fullmap.rs index b6c15e8..80ddb69 100644 --- a/rust/src/fullmap.rs +++ b/rust/src/fullmap.rs @@ -2942,6 +2942,11 @@ fn extract_prebuilt_fullmap_inner( // stale dir from a crashed prior run is removed first, and the dir never // leaks — cleanup is guaranteed on success and best-effort on every error // path (a failed extraction must not leave multi-GB partials behind). + // NOTE: the name is deliberately fixed (not randomized): two CONCURRENT + // extractions to the same output would clobber each other's temp dir, but + // the failure is loud (validation/rename fails -> BABEL fallback) and can + // never land a corrupt bundle — and `build-fullmap` is not run concurrently + // against one output by design. let stem = output .file_stem() .map(|s| s.to_string_lossy().into_owned()) diff --git a/rust/tests/extract_prebuilt.rs b/rust/tests/extract_prebuilt.rs index 68b286a..840cb02 100644 --- a/rust/tests/extract_prebuilt.rs +++ b/rust/tests/extract_prebuilt.rs @@ -190,28 +190,38 @@ fn extract_prebuilt_matches_force_build() { // (the GIL token is required by the signature; `Python::attach` + // `py.detach` inside the function is the production code path). let out_dir = tempfile::tempdir().unwrap(); - let output = out_dir.path().join("fullmap.redb"); + let output = out_dir.path().join("custom.redb"); pyo3::Python::initialize(); pyo3::Python::attach(|py| { tablassert_rs::extract_prebuilt_fullmap(py, archive_path.clone(), output.clone(), None) .unwrap(); }); - // 4. The primary landed at `output` and exactly s0..s15 exist beside it. + // 4. The primary landed at `output` and exactly s0..s15 exist beside it, + // named after the CUSTOM output stem (`custom.s.redb`) — the rename + // contract a `--output my-stem.redb` invocation depends on. assert!(output.exists(), "primary must land at the output path"); for index in 0..common::SHARD_COUNT { let shard = common::shard_path(&output, index); assert!(shard.exists(), "missing extracted shard s{index}"); } + assert_eq!( + common::shard_path(&output, 0) + .file_name() + .unwrap() + .to_string_lossy(), + "custom.s0.redb", + "shards must be renamed after the output stem" + ); assert!( !common::shard_path(&output, common::SHARD_COUNT).exists(), "s16 must not exist" ); - // 5. The temp dir is removed on success: no `.fullmap.prebuilt-extract.d` + // 5. The temp dir is removed on success: no `.custom.prebuilt-extract.d` // (and no dot-prefixed stray at all) may remain in the output dir. assert!( - !out_dir.path().join(".fullmap.prebuilt-extract.d").exists(), + !out_dir.path().join(".custom.prebuilt-extract.d").exists(), "temp dir must be removed after a successful extraction" ); let strays: Vec = std::fs::read_dir(out_dir.path()) @@ -575,3 +585,65 @@ fn named_fullmap_primary_is_preferred_over_strays() { "the preferred `fullmap.redb` primary must land intact" ); } + +/// WHY: the CLI's `section_loop` renders one sub-step line per progress detail +/// from this extractor, and the CHANGELOG documents the exact detail sequence +/// (`opening archive`, `extracting ` per member, `validating`). The +/// callback is best-effort by design (errors discarded), so nothing else would +/// catch a regression that stops firing callbacks or reshapes the details — +/// this test pins the contract end-to-end through the real pyfunction with the +/// GIL released (`py.detach` inside), re-entering Python per callback exactly +/// as production does. The callback is a plain Python list-appending lambda, +/// so no Rust/Python shared state is involved. +#[test] +fn progress_callback_details_are_pinned() { + let fixture_dir = tempfile::tempdir().unwrap(); + let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let archive = fixture_dir.path().join("fullmap.tar.zst"); + let mut members = vec![("fullmap.redb".to_string(), fixture_primary.clone())]; + members.extend(shard_members(&fixture_primary, common::SHARD_COUNT)); + package_tar_zst(&archive, &members); + + pyo3::Python::initialize(); + pyo3::Python::attach(|py| { + use pyo3::types::PyAnyMethods; + + let details = pyo3::types::PyList::empty(py); + let globals = pyo3::types::PyDict::new(py); + globals.set_item("details", &details).unwrap(); + let callback = py + .eval( + c"lambda detail: details.append(detail)", + Some(&globals), + None, + ) + .unwrap(); + + let output = fixture_dir.path().join("out").join("fullmap.redb"); + tablassert_rs::extract_prebuilt_fullmap(py, archive.clone(), output, Some(callback.into())) + .unwrap(); + + let recorded: Vec = details.extract().unwrap(); + assert_eq!( + recorded.first().map(String::as_str), + Some("opening archive"), + "progress must open with the opening-archive detail, got: {recorded:?}" + ); + assert_eq!( + recorded.last().map(String::as_str), + Some("validating"), + "progress must end with the validating detail, got: {recorded:?}" + ); + // One `extracting ` per archive member (17 members: primary + + // 16 shards; `package_tar_zst` writes no directory entries). + let extracting: Vec<&String> = recorded + .iter() + .filter(|detail| detail.starts_with("extracting ")) + .collect(); + assert_eq!( + extracting.len(), + common::SHARD_COUNT + 1, + "one extracting detail per member, got: {recorded:?}" + ); + }); +}