From b6bc69898c723dea61dadad215abac64533f0062 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:15:56 -0500 Subject: [PATCH 01/77] feat(node): integrate the #173 IPFS CID tree-gate onto the #174 walk-concurrency base Rebased onto #174 (fix/served-git-concurrency-cap). #173's incremental history cannot replay commit-by-commit because #174 rewrote the same handler files in parallel; this single commit carries the fully-integrated tree (identical to the verified merge result), with the follow-up fixes as separate commits on top. Brings in #173: GET /ipfs/{cid} CID->oid resolution via pinned_cids, per-caller path-scoped blob/tree/commit-tag gating (#135/#173 F1-F6), and the pin-source provenance table. Adapts #173's tree/commit-tag walks to #174's run_bounded_git so the held /ipfs walk-concurrency permit is duration-safe (F5). --- Cargo.lock | 11 +- crates/gitlawb-node/src/api/ipfs.rs | 1131 +++-- crates/gitlawb-node/src/api/mod.rs | 24 +- crates/gitlawb-node/src/api/repos.rs | 3 + crates/gitlawb-node/src/auth/mod.rs | 5 +- crates/gitlawb-node/src/db/mod.rs | 282 +- crates/gitlawb-node/src/error.rs | 12 + crates/gitlawb-node/src/git/store.rs | 20 + .../gitlawb-node/src/git/visibility_pack.rs | 1079 +++++ crates/gitlawb-node/src/ipfs_pin.rs | 42 +- crates/gitlawb-node/src/main.rs | 17 +- crates/gitlawb-node/src/pinata.rs | 31 +- crates/gitlawb-node/src/rate_limit.rs | 29 + crates/gitlawb-node/src/state.rs | 44 +- crates/gitlawb-node/src/test_support.rs | 3764 +++++++++++++++-- crates/gitlawb-node/src/visibility.rs | 25 + crates/gl/src/ipfs_cmd.rs | 202 +- 17 files changed, 5942 insertions(+), 779 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d12daa43..24beb9ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3300,7 +3300,7 @@ dependencies = [ [[package]] name = "git-remote-gitlawb" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "gitlawb-core", @@ -3312,7 +3312,7 @@ dependencies = [ [[package]] name = "gitlawb-attest" -version = "0.5.0" +version = "0.5.1" dependencies = [ "base64", "ed25519-dalek", @@ -3329,7 +3329,7 @@ dependencies = [ [[package]] name = "gitlawb-core" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "base64", @@ -3356,7 +3356,7 @@ dependencies = [ [[package]] name = "gitlawb-node" -version = "0.5.0" +version = "0.5.1" dependencies = [ "alloy", "anyhow", @@ -3412,7 +3412,7 @@ dependencies = [ [[package]] name = "gl" -version = "0.5.0" +version = "0.5.1" dependencies = [ "alloy", "anyhow", @@ -3824,6 +3824,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sha2", "thiserror 2.0.18", "tracing", ] diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index ac8e615c..a764901b 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -1,14 +1,17 @@ //! GET /ipfs/{cid} — content-addressed retrieval of git objects by CIDv1. //! -//! Every git object stored on this node is addressable by its IPFS CIDv1. +//! Every git object pinned on this node is addressable by its IPFS CIDv1. //! The CID is computed as: //! //! CIDv1(codec=raw, multihash=sha2-256(content_bytes)) //! //! where `content_bytes` is the raw object content as returned by -//! `git cat-file ` (i.e. without the git framing header). -//! This is consistent with how `gitlawb_core::cid::Cid::from_git_object_bytes` -//! computes CIDs when objects are pushed. +//! `git cat-file ` (i.e. without the git framing header) — the +//! same bytes `gitlawb_core::cid::Cid::from_git_object_bytes` hashes when the +//! object is pinned. That digest is NOT the object's git oid: git frames the +//! content with a `" \0"` header before hashing, so `sha2-256(content)` +//! and the git oid differ. The handler therefore maps the CID back to its oid via +//! the `pinned_cids` table rather than treating the digest as an oid (#173). //! //! Serving is access-controlled: an object is returned only from a repo row the //! requesting caller is permitted to read (per-caller path-scoped visibility, @@ -27,14 +30,66 @@ use std::str::FromStr; use crate::auth::AuthenticatedDid; use crate::error::{AppError, Result}; use crate::git::store; -use crate::git::visibility_pack::{allowed_blob_set_for_caller_bounded, has_path_scoped_rule}; +use crate::git::visibility_pack::{ + allowed_blob_set_for_caller_bounded, allowed_tree_set_for_caller_bounded, has_path_scoped_rule, + reachable_commit_tag_oids_bounded, +}; use crate::state::AppState; use crate::visibility::{visibility_check, Decision}; +/// Hard ceiling on the number of full-history reachability walks a single +/// `GET /ipfs/{cid}` request may spawn. The per-request `ipfs_rate_limiter` +/// check brakes *repeat* requests, but within one request the object can exist +/// under path-scoped rules in many repos, and each distinct repo pays its own +/// `spawn_blocking` walk (the memo only dedups the same repo). Without a ceiling +/// a single request fans out to O(repos) walks for one rate-limiter token — an +/// amplification sink (INV-10). Once this many walks have run, no further walk is +/// spawned for the rest of the request: any remaining candidate that still needs +/// a walk is skipped (and, with nothing else readable, the request falls through +/// to the opaque 404). The bound is deliberately generous: a legitimate caller +/// serves on the first repo that grants them, so reaching it requires being +/// denied by this many path-scoped repos first, which real traffic effectively +/// never does. Tunable if that assumption stops holding. +pub(crate) const MAX_HISTORY_WALKS_PER_REQUEST: u32 = 16; + +/// Hard per-request ceiling on how many legacy (NULL-provenance) repositories +/// the CID resolver's scan fallback may PROBE (`acquire` + `git cat-file -t`). +/// The provenance path targets one repo; the legacy scan, absent this bound, +/// fans one anonymous request out to O(repos) subprocess spawns and cold-cache +/// Tigris fetches for a CID enumerable from the public pins index (#173 round 3, +/// F1, INV-10). Deliberately generous: a normal node has far fewer repos than +/// this, so a genuine miss still completes the whole scan and returns a truthful +/// 404; only a node larger than the cap truncates, and a truncated search +/// surfaces as a retryable 503 (never a false "absent"). Legacy pins are a +/// shrinking set — each re-pin backfills provenance — so this fallback is a +/// transitional path, not the steady state. Tunable via `AppState`. +pub(crate) const MAX_LEGACY_PROBES_PER_REQUEST: u32 = 256; + +/// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` buffers and serves +/// (#173 round 8, F6, INV-10). The serve reads via a blocking `git cat-file` and +/// buffers the whole object; unbounded, a large public blob (enumerable from the pins +/// index) could exhaust memory or block a runtime worker. A content-addressed serve +/// must verify the whole object hashes to the requested CID before any byte egresses +/// (F2), so it cannot stream — it buffers up to this cap and withholds anything larger +/// (raise the cap if a class of legitimate objects legitimately exceeds it; never +/// stream unverified). 32 MiB is generous for git blobs/trees/commits. Tunable via +/// `AppState` for the test seam, like the sibling caps. +pub(crate) const MAX_SERVED_OBJECT_BYTES: u64 = 32 * 1024 * 1024; + +/// Lazily-loaded context for the legacy (NULL-provenance) scan fallback in +/// `get_by_cid`: all repos, their visibility rules keyed by repo id, and the set of +/// quarantined repo ids. Loaded once per request only if a legacy pin is hit. +type LegacyScanCtx = ( + Vec, + HashMap>, + HashSet, +); + /// GET /ipfs/{cid} /// -/// Search all repos on the node for a git object whose SHA-256 hash matches -/// the given CIDv1, returning its raw content if the caller may read it. +/// Resolve the CIDv1 to its git oid via the `pinned_cids` table, then search all +/// repos on the node for that object, returning its raw content if the caller may +/// read it. /// /// Visibility (#110, #126): the object is served only from a repo row the /// caller passes. For each iterated row we gate against that row's OWN rules @@ -43,13 +98,16 @@ use crate::visibility::{visibility_check, Decision}; /// row than the one read (KTD2a). We check object existence via /// `store::object_type` *before* the expensive reachability walk so random-CID /// spray cannot trigger full-history git walks on repos that don't carry the -/// object. When the row carries path-scoped rules (KTD4) the served object -/// must be either a non-blob (trees/commits are structural; KTD3) OR a blob -/// in the caller's *reachable* allowed-set (`allowed_blob_set_for_caller`). -/// The reachable allowed-set excludes dangling blobs — a blob written via -/// `git hash-object -w` and never committed has no path to gate, so it is -/// fail-closed 404'd under path-scoped rules (#126). Denial and genuine -/// not-found both fall through to an opaque 404. +/// object. When the row carries path-scoped rules (KTD4) the served object is +/// gated by type: a `blob`/`tree` must be in the caller's *reachable* allowed-set +/// (`allowed_blob_set_for_caller` / `allowed_tree_set_for_caller`), and a +/// `commit`/`tag` must be in the repo's *reachable* commit/tag set +/// (`reachable_commit_tag_oids`, #173). A withheld subtree's tree object is denied +/// here exactly as `get_tree` denies its path, so its child names and oids cannot +/// leak by CID (#135). All these sets exclude dangling objects — a blob, tree, +/// commit, or tag written via plumbing and never referenced has no reachable path, +/// so it is fail-closed 404'd under path-scoped rules (#126, #173). Denial and +/// genuine not-found both fall through to an opaque 404. /// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked @@ -57,14 +115,12 @@ use crate::visibility::{visibility_check, Decision}; pub async fn get_by_cid( Path(cid_str): Path, State(state): State, - auth: Option>, - // Per-source keying for the walk concurrency sub-cap. Infallible extractors - // (mirror the git handlers in `repos.rs`): `PeerAddr` yields `None` under - // `oneshot` with no `ConnectInfo`, and the header map falls back per `client_key`. crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, - req_headers: HeaderMap, + headers: HeaderMap, + auth: Option>, ) -> Result { - // 1. Decode the CID and extract the SHA-256 digest + // 1. Decode and validate the CID (uniform 400 on a malformed / non-sha2-256 + // CID, before any DB or git work). let cid = CidGeneric::<64>::from_str(&cid_str) .map_err(|e| AppError::BadRequest(format!("invalid CID: {e}")))?; @@ -77,9 +133,13 @@ pub async fn get_by_cid( )); } - let sha256_hex = hex::encode(mh.digest()); - let caller = auth.as_ref().map(|e| e.0 .0.as_str()); - let caller_owned = caller.map(|c| c.to_string()); + // Canonicalize the CID for the pinned_cids lookup. Pins are stored under the + // canonical base32 `cid.to_string()`, but a client may send any equivalent + // multibase spelling (base58/base64) of the same CID; those parse and pass + // the sha2-256 check yet miss the canonical key, so they must be normalized + // before the DB lookup (#173). Response headers and error messages still echo + // the original `cid_str` the client sent. + let canonical_cid = cid.to_string(); // Bounded walk admission (#174 P1-3), taken before any DB/git work so a flood sheds // cheaply. The per-repo `spawn_blocking` walk below is a full-history git walk with @@ -87,13 +147,16 @@ pub async fn get_by_cid( // out concurrent walks past every git pool, exhausting the blocking pool + PIDs. // Acquire the global permit (and, for a resolvable source, the per-source // sub-permit) ONCE here and hold BOTH for the whole request — across every - // `spawn_blocking` walk in the loop below — so the slot reflects real blocking-thread + // `spawn_blocking` walk below — so the slot reflects real blocking-thread // occupancy (a tokio walk-timeout cannot free it while the blocking work still runs) - // and one request cannot open more than its share of concurrent walks. On - // unavailability shed a clean 503. The per-source key is the resolved source IP - // (`client_key`), never the DID (`/ipfs` admits any `did:key` unthrottled, so a DID - // key would be free to mint around); a `None` key (no trusted header, no peer) is - // bounded by the global pool only, never the per-source sub-cap. + // and one request cannot open more than its share of concurrent walks. Holding a + // slot across a walk is only safe because every walk child is duration-bounded + // (`*_bounded` + `run_bounded_git` teardown), so a hung git cannot pin the slot + // past `git_service_timeout_secs`. On unavailability shed a clean 503. The + // per-source key is the resolved source IP (`client_key`), never the DID (`/ipfs` + // admits any `did:key` unthrottled, so a DID key would be free to mint around); a + // `None` key (no trusted header, no peer) is bounded by the global pool only, + // never the per-source sub-cap. let _ipfs_walk_permit = state .git_ipfs_walk_semaphore .clone() @@ -102,7 +165,7 @@ pub async fn get_by_cid( tracing::warn!("/ipfs walk concurrency cap reached; shedding request with 503"); AppError::Overloaded("ipfs service at capacity, retry shortly".into()) })?; - let source_key = crate::rate_limit::client_key(&req_headers, peer, state.push_limiter_trust); + let source_key = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust); let _ipfs_caller_permit = match &source_key { Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); @@ -111,178 +174,553 @@ pub async fn get_by_cid( None => None, }; - // 2. Search all repos for an object with this SHA-256 - let repos = state + // Resolve the content-addressed CID to the object's git oid(s). A real pin + // CID digests the raw object content (`Cid::from_git_object_bytes`), NOT the + // git oid (git frames content with a `" \0"` header first), so we + // map it back through `pinned_cids` rather than treating the digest as an oid + // (#173). The cid index is non-unique, so one CID can map to several oids (a + // tree and a blob whose raw bytes collide, or content pinned under two oids); + // we try each candidate below rather than pick one arbitrarily and false-404 + // when the chosen one is withheld or absent while another is readable (#173). + // An empty result is an opaque 404, uniform with a genuine not-found and a + // visibility denial. + let oids = state .db - .list_all_repos() + .oids_for_cid(&canonical_cid) .await .map_err(AppError::Internal)?; + if oids.is_empty() { + return Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))); + } + let caller = auth.as_ref().map(|e| e.0 .0.as_str()); + let caller_owned = caller.map(|c| c.to_string()); - // Fetch every repo's visibility rules in one query rather than one per row - // (the gate runs each row against its OWN rules — KTD2a). A row absent from - // the map has no rules. - let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); - let rules_by_repo = state - .db - .list_visibility_rules_for_repos(&repo_ids) - .await - .map_err(AppError::Internal)?; + // Per-request walk budget + memos + throttle flag, shared by the provenance path + // and the legacy scan so both honor the same fan-out ceiling, per-repo memo, and + // IP brake. The caller is constant for one request, so `repo.id` alone keys the memo. + let mut walk = WalkState { + walks: 0, + probes: 0, + truncated: false, + allowed_blob_memo: HashMap::new(), + allowed_tree_memo: HashMap::new(), + reachable_ct_memo: HashMap::new(), + }; + // Set when a walk-requiring candidate is skipped because the source IP's walk quota + // is spent (#173 review, F-C): the scan keeps going so a later walk-free copy still + // serves; only if nothing is servable is it turned into the 429. + let mut throttled = false; + let rctx = ResolveCtx { + caller, + caller_owned: &caller_owned, + headers: &headers, + peer, + cid_str: &cid_str, + canonical_cid: &canonical_cid, + }; - // Request-scoped memo of the per-repo allowed-blob set (KTD1, #126). The - // caller is constant for one request, so `repo.id` alone is a safe, - // sufficient key — never a coarse caller "class", which - // `visibility_check`'s exact full-DID reader match would make unsafe. - // - // We flipped from a deny-set (`withheld_blob_oids`) to an allowed-set - // (`allowed_blob_set_for_caller`) so dangling blobs — never enumerated by - // the reachable walk — fail closed instead of slipping through an empty - // deny entry (#126). - let mut allowed_memo: HashMap> = HashMap::new(); - - // Cap the number of candidate repos one request walks (it already short-circuits on - // serve): a CID present in — or path-gated out of — many repos must not serialize an - // unbounded number of full-history walks inside the single held admission slot. - let mut repos_walked: usize = 0; - - for repo in &repos { - // Repo-level read gate against THIS row's own rules (KTD2a). - let rules: &[crate::db::VisibilityRule] = rules_by_repo - .get(&repo.id) - .map(Vec::as_slice) - .unwrap_or(&[]); - if visibility_check(rules, repo.is_public, &repo.owner_did, caller, "/") == Decision::Deny { - continue; - } + // Legacy scan context (repos + rules + quarantined ids), loaded LAZILY only when a + // legacy NULL-provenance pin is hit — the provenance path must never trigger the + // O(repos) load (that fan-out is exactly what provenance removes, #173 round 2). + let mut scan_ctx: Option = None; - // Loop bound (#174 P1-3): once this request has walked its cap of candidate - // repos (each a git subprocess, up to a full-history walk), stop and fall - // through to the opaque 404 rather than serialize an unbounded number under the - // single held admission slot. - if repos_walked >= state.config.ipfs_max_repos_walked { - tracing::warn!( - cap = state.config.ipfs_max_repos_walked, - "/ipfs request hit the per-request repo-walk cap; stopping the scan" - ); - break; - } - repos_walked += 1; - - // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares - // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise - // block the whole /ipfs request). On expiry keep the existing fail-closed skip — - // never serve an un-acquired repo; a public copy (if any) still gets its turn. - let acquire_deadline = - std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); - let repo_path = match tokio::time::timeout( - acquire_deadline, - state.repo_store.acquire(&repo.owner_did, &repo.name), - ) - .await - { - Ok(Ok(p)) => p, - Ok(Err(_)) => continue, - Err(_elapsed) => { - tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs walk; skipping repo"); - continue; + for sha256_hex in &oids { + // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). + // Resolve a PROVENANCED pin by trying each source repo (bounded to + // MAX_PIN_SOURCES) through the SAME gate; the first that authorizes serves — no + // scan fan-out. A shared object first pinned from a private/quarantined repo + // still serves from a later PUBLIC source. Deterministic (ORDER BY on the + // union), so no ordering can turn an authorized copy into a 404. + let sources = state + .db + .pin_sources_for_oid(sha256_hex) + .await + .map_err(AppError::Internal)?; + if sources.is_empty() { + // F3 (#173, INV-10/INV-15): the legacy NULL-provenance scan builds an + // O(repos) preload (repos + rules + quarantine) BEFORE gate_and_serve's + // per-probe brake can bite, so a throttled source could still force O(repos) + // DB work on every replay. Peek the per-IP limiter WITHOUT consuming a token: + // an already-throttled source is shed here, before the preload runs. The + // consuming per-probe charge inside gate_and_serve is left UNCHANGED (it is + // load-bearing for the across-request bound), so this adds no double-charge — + // a non-consuming peek plus the existing per-probe charge, never two charges. + if let Some(key) = + crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) + { + if state.ipfs_rate_limiter.is_throttled(&key).await { + throttled = true; + continue; + } } - }; - - // Check whether the object exists in this repo before any expensive - // reachability walk. This prevents random-CID spray from triggering - // full-history git walks on repos that don't carry the object. - let obj_type = match store::object_type(&repo_path, &sha256_hex) { - Ok(Some(t)) => t, - Ok(None) => continue, - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); - continue; + // Legacy pin (recorded before provenance existed): fall back to the repo + // scan, gating each repo through the SAME gate. Load the scan context + // once, lazily. + if scan_ctx.is_none() { + #[cfg(test)] + bump_preload_queries(); + let repos = state + .db + .list_all_repos() + .await + .map_err(AppError::Internal)?; + let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); + let rules_by_repo = state + .db + .list_visibility_rules_for_repos(&repo_ids) + .await + .map_err(AppError::Internal)?; + let quarantined: HashSet = state + .db + .list_quarantined_repos() + .await + .map_err(AppError::Internal)? + .into_iter() + .map(|r| r.id) + .collect(); + scan_ctx = Some((repos, rules_by_repo, quarantined)); } - }; - - // Per-blob gating only applies when a path-scoped rule exists (KTD4). - // Without any path-scoped rule, the "/" gate above is the whole story. - // Trees/commits are always served under path-scoped rules (KTD3). - let path_scoped = has_path_scoped_rule(rules); - if path_scoped && obj_type == "blob" { - if !allowed_memo.contains_key(&repo.id) { - let rp = repo_path.clone(); - let r = rules.to_vec(); - let is_public = repo.is_public; - let owner = repo.owner_did.clone(); - let caller_for_walk = caller_owned.clone(); - let git_bin = state.git_bin.clone(); - let walk_timeout = - std::time::Duration::from_secs(state.config.git_service_timeout_secs); - // Full-history walk shells out to git — keep it off the async runtime, - // bounded and reaped like the served-git ops (#174). - let walk = tokio::task::spawn_blocking(move || { - allowed_blob_set_for_caller_bounded( - &rp, - &git_bin, - walk_timeout, - &r, - is_public, - &owner, - caller_for_walk.as_deref(), - ) - }) - .await; - // Fail closed on EITHER a task panic (JoinError) or a walk error: - // we cannot prove the caller may read here, so skip this repo and - // let a public copy (if any) serve. Never serve on an unproven gate. - let set = match walk { - Ok(Ok(set)) => set, - Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk failed; skipping repo"); - continue; - } - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk task panicked; skipping repo"); + let (repos, rules_by_repo, quarantined) = scan_ctx.as_ref().unwrap(); + for repo in repos { + let rules = rules_by_repo + .get(&repo.id) + .map(Vec::as_slice) + .unwrap_or(&[]); + let is_quar = quarantined.contains(&repo.id); + match gate_and_serve( + &state, repo, rules, is_quar, sha256_hex, &rctx, &mut walk, true, + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + // A throttled walk-requiring candidate is skipped, not fatal: + // keep scanning for a later walk-free copy (#173 review, F-C). + GateOutcome::Throttled => throttled = true, + GateOutcome::Skip => {} + } + } + } else { + for repo_id in &sources { + let repo = match state + .db + .get_repo_by_id(repo_id) + .await + .map_err(AppError::Internal)? + { + Some(r) => r, + // A source repo is gone: skip this source. Do NOT fall back to the + // scan (that would reopen the fan-out); a later source or oid + // candidate may still resolve. + None => continue, + }; + let quarantined = state + .db + .is_repo_quarantined(repo_id) + .await + .map_err(AppError::Internal)?; + let rules_map = state + .db + .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) + .await + .map_err(AppError::Internal)?; + let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); + match gate_and_serve( + &state, + &repo, + rules, + quarantined, + sha256_hex, + &rctx, + &mut walk, + false, // provenance path: bounded source set, no scan fan-out + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + GateOutcome::Throttled => { + throttled = true; continue; } - }; - allowed_memo.insert(repo.id.clone(), set); + GateOutcome::Skip => continue, + } } - let in_allowed = allowed_memo - .get(&repo.id) - .is_some_and(|set| set.contains(&sha256_hex)); - if !in_allowed { - continue; + } + } + + // Nothing served — three distinct tails, in precedence order: + // 1. The scan was cut short by a cap (legacy probe ceiling or walk ceiling), so + // the object was NOT proven absent/unreadable everywhere → 503, retryable, and + // explicitly NOT a definitive not-found (#173, F2). This outranks the throttle: + // an incomplete search must not masquerade as a clean rate-limit outcome, and + // it carries only the caller-supplied CID (no object/OID/metadata leak). + // 2. A walk-requiring candidate was skipped for a spent IP quota while the scan + // otherwise completed → 429 (the brake bit; a cheaper copy was sought first). + // 3. A full scan under the caps found nothing readable → opaque 404, uniform with + // a genuine not-found and a visibility denial. + if walk.truncated { + return Err(AppError::SearchIncomplete(format!( + "CID {cid_str} search incomplete — retry" + ))); + } + if throttled { + return Err(AppError::TooManyRequests( + "ipfs retrieval rate limit exceeded — try again later".into(), + )); + } + Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))) +} + +/// Outcome of gating one repo for one candidate oid. +enum GateOutcome { + /// The object passed the gate; serve this response. + Served(Response), + /// This repo does not serve the object (absent, denied, quarantined, walk-capped, + /// or a walk error) — try the next candidate. + Skip, + /// A walk-requiring candidate hit the per-IP walk quota; skip it but let the caller + /// record the throttle so a later walk-free copy can still serve. + Throttled, +} + +/// Outcome of the bounded, off-worker object read for one gated candidate (F6, #173). +enum ServedRead { + /// Verified: the object's bytes hash to the requested CID; serve them. + Ok(Vec), + /// The bytes do not hash to the requested CID (a legacy provider-CID row); withhold. + Mismatch(String), + /// The object exceeds the served-object size cap; withhold rather than buffer it. + TooLarge(u64), + /// The object is genuinely absent (git reported it does not exist); try the next + /// candidate. Distinct from `ReadErr` so an infra failure is never silently rendered + /// as a clean not-found. + Gone, + /// A git subprocess failed to run (spawn/IO error, not a "no such object"). Logged at + /// the handler layer and skipped — an infra failure must surface as an error, not a + /// silent 404 for an authorized caller (INV-25 spirit, #173). + ReadErr(String), +} + +/// Immutable per-request context threaded into the gate. +struct ResolveCtx<'a> { + caller: Option<&'a str>, + caller_owned: &'a Option, + headers: &'a HeaderMap, + peer: Option, + cid_str: &'a str, + /// Canonical base32 form of the requested CID (`cid.to_string()`), used by the + /// serve-side integrity check to confirm the served bytes actually hash to the + /// requested content address (F2, #173). Compared against the recomputed CID, NOT + /// `cid_str` — a client may send an equivalent non-canonical multibase spelling. + canonical_cid: &'a str, +} + +/// Per-request walk budget + memos, shared across the provenance path and the legacy +/// scan so the fan-out ceiling and per-repo memoization span the whole request. +struct WalkState { + walks: u32, + /// Count of legacy (NULL-provenance) repos actually probed this request, so the + /// scan can stop at `ipfs_max_legacy_probes` instead of fanning out to O(repos) + /// `acquire` + `cat-file` (#173, F1, INV-10). Only the legacy path bumps it. + probes: u32, + /// Set when any cap (the legacy probe ceiling or the walk ceiling) cut the scan + /// short. A truncated scan did NOT prove the object absent/unreadable everywhere, + /// so the tail returns a retryable 503 rather than a definitive 404 (#173, F2). + truncated: bool, + allowed_blob_memo: HashMap>, + allowed_tree_memo: HashMap>, + reachable_ct_memo: HashMap>, +} + +/// Gate ONE repo for ONE candidate oid and, if the caller may read it, serve it. The +/// SINGLE gate both the provenance path and the legacy scan call, so INV-11 (quarantine +/// hard-drops before visibility), INV-2 (the repo's own "/" gate), and the per-object +/// reachability walk hold identically on both paths (KTD5). Never re-resolves via +/// `authorize_repo_read`, whose fuzzy match could authorize a different physical row +/// than the one read (KTD2a). +// The per-repo gate genuinely needs the row, its rules, its quarantine bit, the oid, +// the request context, the shared walk budget, and whether this is the fan-out-bounded +// legacy scan; bundling them buys nothing over the existing threshold. +#[allow(clippy::too_many_arguments)] +async fn gate_and_serve( + state: &AppState, + repo: &crate::db::RepoRecord, + rules: &[crate::db::VisibilityRule], + quarantined: bool, + sha256_hex: &str, + ctx: &ResolveCtx<'_>, + walk: &mut WalkState, + // True only for the legacy NULL-provenance scan, which iterates every repo. The + // provenance path targets one repo (no fan-out) and passes false, so it does not + // consume the per-request probe budget below. + legacy_scan: bool, +) -> GateOutcome { + // Quarantine gate (INV-11): a quarantined mirror is hidden from every reader, owner + // included, BEFORE any visibility check — so an owner whom visibility would Allow + // still 404s. + if quarantined { + return GateOutcome::Skip; + } + // Repo-level "/" read gate against THIS row's own rules (INV-2, KTD2a). + if visibility_check(rules, repo.is_public, &repo.owner_did, ctx.caller, "/") == Decision::Deny { + return GateOutcome::Skip; + } + // Legacy-scan fan-out control (#173, F1/F3, INV-10). The legacy path probes every + // root-visible repo, and the probe below (`acquire` — a possible cold-cache + // Tigris fetch — plus a `git cat-file -t` subprocess) is the expensive part. + // Cap it per request BEFORE that work runs, so an anonymous caller wielding a + // CID from the public pins index cannot amplify one request into O(repos) + // subprocesses. A legacy scan is inherently fan-out (unlike a targeted + // provenance fetch), so EVERY legacy probe is charged to the source IP from the + // first one, not just the ones past a free budget. A per-request-only budget + // reset each request, leaving a NULL-provenance CID open to unbounded ACROSS- + // request amplification: N requests spending N x budget cold `acquire` calls + // against Tigris with zero limiter contact (#173, F3, jatmn). Charging the first + // probe makes those requests accumulate against the per-IP `ipfs_rate_limiter`, + // closing that path. The per-request cap below stays as the second bound (a + // single request's ceiling). A spent quota is the same non-fatal Throttled as the + // walk brake: keep scanning for a walk-free copy, and only a wholly-unservable + // request becomes the 429. No resolvable key (a test oneshot with no peer/header) + // skips the brake, as the walk brake does. The provenance path targets one repo + // (no fan-out) and is exempt (`legacy_scan == false`). + if legacy_scan { + if walk.probes >= state.ipfs_max_legacy_probes { + // Budget spent: stop probing and mark the scan truncated so the tail + // reports an incomplete search (503), not a false 404 (#173, F2). + walk.truncated = true; + return GateOutcome::Skip; + } + if let Some(key) = + crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) + { + if !state.ipfs_rate_limiter.check(&key).await { + return GateOutcome::Throttled; } } + walk.probes += 1; + } + let repo_path = match state.repo_store.acquire(&repo.owner_did, &repo.name).await { + Ok(p) => p, + Err(_) => return GateOutcome::Skip, + }; - // Now that we've passed the gate, read the content. - let content = match store::read_object_content(&repo_path, &sha256_hex, &obj_type) { - Ok(c) => c, + // Existence probe before any walk (random-CID spray must not trigger a walk on a + // repo that lacks the object). Off the async runtime — it shells out to + // `git cat-file -t`. Fail closed (skip) on a task panic. + let obj_type = { + let rp = repo_path.clone(); + let sha = sha256_hex.to_string(); + match tokio::task::spawn_blocking(move || store::object_type(&rp, &sha)).await { + Ok(Ok(Some(t))) => t, + Ok(Ok(None)) => return GateOutcome::Skip, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); + return GateOutcome::Skip; + } Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); - continue; + tracing::warn!(repo = %repo.name, err = %e, "object-type probe task panicked; skipping repo"); + return GateOutcome::Skip; } - }; + } + }; - // 3. Return the content with IPFS-style headers - let mut headers = HeaderMap::new(); - headers.insert( - HeaderName::from_static("content-type"), - HeaderValue::from_static("application/octet-stream"), - ); - headers.insert( - HeaderName::from_static("x-content-cid"), - HeaderValue::from_str(&cid_str).unwrap_or_else(|_| HeaderValue::from_static("invalid")), - ); - headers.insert( - HeaderName::from_static("x-git-hash"), - HeaderValue::from_str(&sha256_hex) - .unwrap_or_else(|_| HeaderValue::from_static("invalid")), - ); + // Per-object gating applies only under a path-scoped rule (KTD4); otherwise the "/" + // gate above is the whole story. A blob is gated on the caller's allowed-blob set, a + // tree on the allowed-tree set (#135), a commit/tag on the repo's reachable + // commit/tag set (#173) — each a full-history walk sharing the per-request cap and + // per-walk IP quota. + let path_scoped = has_path_scoped_rule(rules); + let gated = path_scoped && matches!(obj_type.as_str(), "blob" | "tree" | "commit" | "tag"); + if gated { + let already = match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.contains_key(&repo.id), + "tree" => walk.allowed_tree_memo.contains_key(&repo.id), + "commit" | "tag" => walk.reachable_ct_memo.contains_key(&repo.id), + other => unreachable!("gated admits only blob/tree/commit/tag, got {other}"), + }; + if !already { + // Per-request fan-out ceiling (INV-10): once this many walks have run, skip + // THIS walk-requiring candidate and keep scanning (a later walk-free copy + // must still serve). `walks` is bumped only inside this block, so walk-free + // candidates never consume budget. + if walk.walks >= state.ipfs_max_history_walks { + // The walk ceiling truncated the search: a later repo (possibly one that + // authorizes this caller) is left unwalked, so absence is unproven — + // record it so the tail returns 503, not a false 404 (#173, F2). + walk.truncated = true; + return GateOutcome::Skip; + } + // Brake each spawned walk on the source IP (#173, F3, INV-15), BEFORE + // spending walk budget: a throttled candidate neither walks nor consumes + // budget and must not end the request — skip it and keep scanning + // (#173 review, F-C). No key (a test oneshot with no peer/header) skips the + // brake, as the other IP brakes do. On the LEGACY path the probe brake + // above already charged THIS candidate to the source (#173, F3, jatmn), so + // the walk brake must not double-charge it: only the provenance path + // (`legacy_scan == false`, no probe toll) charges here. + if !legacy_scan { + if let Some(key) = + crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) + { + if !state.ipfs_rate_limiter.check(&key).await { + return GateOutcome::Throttled; + } + } + } + walk.walks += 1; - return Ok((StatusCode::OK, headers, content).into_response()); + let rp = repo_path.clone(); + let r = rules.to_vec(); + let is_public = repo.is_public; + let owner = repo.owner_did.clone(); + let caller_for_walk = ctx.caller_owned.clone(); + let kind = obj_type.clone(); + // Every walk is the DURATION-BOUNDED twin (`run_bounded_git` teardown under + // `git_service_timeout_secs`): the handler holds its /ipfs walk permit + // across this spawn_blocking, and a held permit is only safe if no walk + // child can outlive the deadline (#174 F5). + let git_bin = state.git_bin.clone(); + let walk_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let result = tokio::task::spawn_blocking(move || match kind.as_str() { + "blob" => allowed_blob_set_for_caller_bounded( + &rp, + &git_bin, + walk_timeout, + &r, + is_public, + &owner, + caller_for_walk.as_deref(), + ), + "tree" => allowed_tree_set_for_caller_bounded( + &rp, + &git_bin, + walk_timeout, + &r, + is_public, + &owner, + caller_for_walk.as_deref(), + ), + "commit" | "tag" => reachable_commit_tag_oids_bounded(&rp, &git_bin, walk_timeout), + other => unreachable!("gated admits only blob/tree/commit/tag, got {other}"), + }) + .await; + // Fail closed on a walk error or task panic: we cannot prove readability, so + // skip rather than serve on an unproven gate. + let set = match result { + Ok(Ok(set)) => set, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "allowed-set walk failed; skipping repo"); + return GateOutcome::Skip; + } + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "allowed-set walk task panicked; skipping repo"); + return GateOutcome::Skip; + } + }; + match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.insert(repo.id.clone(), set), + "tree" => walk.allowed_tree_memo.insert(repo.id.clone(), set), + _ => walk.reachable_ct_memo.insert(repo.id.clone(), set), + }; + } + let in_set = match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.get(&repo.id), + "tree" => walk.allowed_tree_memo.get(&repo.id), + _ => walk.reachable_ct_memo.get(&repo.id), + } + .is_some_and(|set| set.contains(sha256_hex)); + if !in_set { + return GateOutcome::Skip; + } } - // Not found in any repo - Err(AppError::RepoNotFound(format!( - "no git object found for CID {cid_str}" - ))) + // Passed the gate — bound the object, read it OFF the async worker, and verify the + // content address, all before any byte egresses. F6 (#173): read_object_content runs a + // blocking `git cat-file` and buffers the whole object; called directly on the Axum + // worker (the type-probe and walk are already off-worker) it blocks a runtime thread, + // and unbounded it can exhaust memory for a large public blob (enumerable from the pins + // index). Precheck the SIZE and run size + read + verify inside spawn_blocking. A + // content-addressed serve cannot verify a STREAMED body (the digest is known only after + // the last byte, by which point the prefix has already egressed), so we never stream: + // buffer-verify-then-serve up to the cap and withhold anything larger. F2's integrity + // check moves in here too, so no unverified bytes are ever assembled into a response. + let max_bytes = state.ipfs_max_served_object_bytes; + let read_repo = repo_path.clone(); + let read_sha = sha256_hex.to_string(); + let read_type = obj_type.clone(); + let want_cid = ctx.canonical_cid.to_string(); + let read = tokio::task::spawn_blocking(move || -> ServedRead { + match store::object_size(&read_repo, &read_sha) { + Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), + Ok(Some(_)) => {} + // git ran and reported no such object (or an unparseable size): genuine + // not-found for this candidate. + Ok(None) => return ServedRead::Gone, + // git itself failed to run: an infra failure, not a not-found. + Err(e) => return ServedRead::ReadErr(e.to_string()), + } + let content = match store::read_object_content(&read_repo, &read_sha, &read_type) { + Ok(c) => c, + Err(e) => return ServedRead::ReadErr(e.to_string()), + }; + let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); + if served != want_cid { + return ServedRead::Mismatch(served); + } + ServedRead::Ok(content) + }) + .await; + let content = match read { + Ok(ServedRead::Ok(c)) => c, + Ok(ServedRead::TooLarge(size)) => { + tracing::warn!( + repo = %repo.name, size, max = max_bytes, + "withholding object: exceeds the served-object size cap (F6)" + ); + #[cfg(test)] + note_oversize_reject(); + return GateOutcome::Skip; + } + Ok(ServedRead::Mismatch(served)) => { + tracing::warn!( + repo = %repo.name, requested = %ctx.canonical_cid, served = %served, + "withholding object: served bytes do not hash to the requested CID (legacy provider-CID row?)" + ); + return GateOutcome::Skip; + } + Ok(ServedRead::Gone) => return GateOutcome::Skip, + Ok(ServedRead::ReadErr(e)) => { + // Infra failure (git spawn/IO), NOT a not-found: mark the search truncated so + // a wholly-unserved request tails to a retryable 503, never a definitive 404 + // for an authorized caller (INV-25 spirit — logging alone is not surfacing). + tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); + walk.truncated = true; + return GateOutcome::Skip; + } + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "object read task panicked"); + walk.truncated = true; + return GateOutcome::Skip; + } + }; + let mut resp_headers = HeaderMap::new(); + resp_headers.insert( + HeaderName::from_static("content-type"), + HeaderValue::from_static("application/octet-stream"), + ); + resp_headers.insert( + HeaderName::from_static("x-content-cid"), + HeaderValue::from_str(ctx.cid_str).unwrap_or_else(|_| HeaderValue::from_static("invalid")), + ); + resp_headers.insert( + HeaderName::from_static("x-git-hash"), + HeaderValue::from_str(sha256_hex).unwrap_or_else(|_| HeaderValue::from_static("invalid")), + ); + GateOutcome::Served((StatusCode::OK, resp_headers, content).into_response()) } /// GET /api/v1/ipfs/pins @@ -303,6 +741,58 @@ pub async fn list_pins(State(state): State) -> Result = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_preload_queries() { + PRELOAD_QUERIES.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn preload_queries() -> usize { + PRELOAD_QUERIES.with(|c| c.get()) +} + +#[cfg(test)] +fn bump_preload_queries() { + PRELOAD_QUERIES.with(|c| c.set(c.get() + 1)); +} + +// Test-only INV-10 cost counter (F6, U6/U7): how many times the serve path withheld an +// object because it exceeded `ipfs_max_served_object_bytes`. The bounded read must reject +// an oversized object rather than buffer it on the worker; the counter is the both-ways +// guard (a removed size precheck stops incrementing it and serves the oversized object). +// Set from the match arm after `spawn_blocking` resolves, i.e. on the test's runtime +// thread, so the thread-local is read on the same thread it is written. +#[cfg(test)] +thread_local! { + static OVERSIZE_REJECTS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_oversize_rejects() { + OVERSIZE_REJECTS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn oversize_rejects() -> usize { + OVERSIZE_REJECTS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_oversize_reject() { + OVERSIZE_REJECTS.with(|c| c.set(c.get() + 1)); +} + #[cfg(test)] mod tests { //! #174 P1-3 (U3): the public `GET /ipfs/{cid}` walk carries bounded CONCURRENCY @@ -310,7 +800,9 @@ mod tests { //! walk, plus a per-IP route rate limit. These are handler-layer proofs: mount the //! real handler/router, drive one request, assert the exact 503 shed, then name the //! mutation that turns each RED. The per-source key resolves an IP only (`Some(ip)` - //! vs `None`), never a DID — both arms are driven so neither is vacuous. + //! vs `None`), never a DID — both arms are driven so neither is vacuous. The + //! CID-resolution / visibility-gate behavior of the handler itself is covered by the + //! `#[sqlx::test]` suite in `test_support.rs`. use axum::body::Body; use axum::extract::ConnectInfo; @@ -500,29 +992,33 @@ mod tests { drop(held); } - /// Retain-through-blocking (R3, the load-bearing async property): the walk - /// admission is held until the `spawn_blocking` walk actually RETURNS, not when a - /// tokio timeout fires. With the global pool at size 1, drive a request until its - /// walk (a fake git that hangs on `rev-list`) is in flight; the slot must stay held - /// (`available_permits() == 0`) and a replacement from a DIFFERENT source must shed - /// 503 for as long as the blocking walk runs — even though the request future is - /// only `.await`ing the blocking join. When the blocking walk ends the permit frees - /// and a replacement is admitted. The permit lives INSIDE the handler across the - /// blocking `.await`; move it out (drop before the walk) and the replacement would - /// be admitted while the walk still burns a blocking thread (the bug this guards). + /// Retain-through-blocking (#174 F5, the load-bearing async property, on the + /// NEWLY-BOUNDED TREE path): the walk admission is held until the `spawn_blocking` + /// walk actually RETURNS, not when a tokio timeout fires. The requested CID + /// resolves to a TREE object under a path-scoped rule, so the gate runs + /// `allowed_tree_set_for_caller_bounded` — the walk this integration converts to + /// `run_bounded_git` — rather than the blob walk #174 already proved. With the + /// global pool at size 1, drive a request until its walk (a fake git that hangs on + /// `rev-list`) is in flight; the slot must stay held (`available_permits() == 0`) + /// and a replacement from a DIFFERENT source must shed 503 for as long as the + /// blocking walk runs — even though the request future is only `.await`ing the + /// blocking join. When the blocking walk ends the permit frees and a replacement + /// is admitted. The permit lives INSIDE the handler across the blocking `.await`; + /// move it out (drop before the walk) and the replacement would be admitted while + /// the walk still burns a blocking thread (the bug this guards). #[cfg(unix)] #[sqlx::test] - async fn get_by_cid_walk_permit_held_through_blocking_walk(pool: sqlx::PgPool) { + async fn get_by_cid_walk_permit_held_through_bounded_tree_walk(pool: sqlx::PgPool) { use std::process::Command; let tmp = tempfile::TempDir::new().unwrap(); let revlist_pid = tmp.path().join("revlist.pid"); - // Fake git for the /ipfs WALK only (object_type/read_object_content use the real - // `git`, so the object must genuinely exist below). Empty refs (so - // assert_all_refs_are_commits returns Ok without the peel), `rev-parse` resolves, - // and `rev-list` records its pid then sleeps ~6s so the walk BLOCKS - // deterministically. The sleep bounds the walk so a broken fix cannot wedge the - // suite. + // Fake git for the /ipfs TREE walk only (object_type/read_object_content use + // the real `git`, so the tree must genuinely exist below). `rev-parse` + // resolves (so the lenient enumeration appends HEAD) and `rev-list` records + // its pid then sleeps ~6s so the walk BLOCKS deterministically inside + // `run_bounded_git`. The sleep bounds the walk so a broken fix cannot wedge + // the suite. let body = format!( "#!/bin/sh\n\ case \"$1\" in\n\ @@ -554,18 +1050,18 @@ mod tests { state.git_bin = git_path.to_str().unwrap().to_string(); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; - let owner = "z6ipfs1"; - let name = "ip1"; + let owner = "z6ipfstree"; + let name = "iptree"; state .db .upsert_mirror_repo(owner, name, "/unused", None, false) .await .unwrap(); let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); - // The exact bare path the handler's `acquire` resolves. Build a REAL SHA-256 bare - // repo there with a committed blob under `src/`, so real `git cat-file -t ` classifies it as a blob (the CID digest IS the sha256 object id in - // object-format=sha256) and the handler reaches the path-scoped walk branch. + // The exact bare path the handler's `acquire` resolves. Build a REAL SHA-256 + // bare repo there with a committed `src/` directory, so real + // `git cat-file -t ` classifies the requested object as a TREE and the + // handler routes into the tree-walk arm of the gate. let bare = state .repo_store .acquire(&rec.owner_did, &rec.name) @@ -587,7 +1083,11 @@ mod tests { }; let work = tmp.path().join("work"); std::fs::create_dir_all(work.join("src")).unwrap(); - std::fs::write(work.join("src/secret.txt"), b"ipfs walk retain proof\n").unwrap(); + std::fs::write( + work.join("src/secret.txt"), + b"ipfs tree walk retain proof\n", + ) + .unwrap(); run( &["init", "-q", "--object-format=sha256", "-b", "main"], &work, @@ -606,38 +1106,46 @@ mod tests { ], tmp.path(), ); - // The blob's SHA-256 object id (= the CID's digest); build the CID from it. - let oid = { + // The `src` directory's TREE oid — the object the request asks for. + let tree_oid = { let out = Command::new("git") - .args(["rev-parse", "HEAD:src/secret.txt"]) + .args(["rev-parse", "HEAD:src"]) .current_dir(&work) .output() .expect("git rev-parse runs"); assert!(out.status.success(), "rev-parse failed"); String::from_utf8_lossy(&out.stdout).trim().to_string() }; - let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); - let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) - .as_str() - .to_string(); - // Precondition: real git classifies the object as a blob (so the handler reaches - // the walk branch, not an early `continue`). + // Precondition: real git classifies the object as a TREE (so the handler + // reaches the tree-walk arm, not the blob arm or an early `continue`). assert_eq!( - crate::git::store::object_type(&bare, &oid) + crate::git::store::object_type(&bare, &tree_oid) .unwrap() .as_deref(), - Some("blob"), - "the seeded sha256 blob must exist so the handler reaches the walk" + Some("tree"), + "the seeded sha256 tree must exist so the handler reaches the tree walk" ); - // A path-scoped rule so has_path_scoped_rule() is true (the walk branch) without - // denying the "/" gate on the public repo. + // Pin the tree's content CID WITH provenance so the resolver targets this one + // repo (no legacy scan). A real pin CID digests the raw object content, not + // the git oid, so build it exactly as the pin path does (#173). + let (_ty, raw) = crate::git::store::read_object(&bare, &tree_oid) + .unwrap() + .expect("tree object readable"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + state + .db + .record_pinned_cid(&tree_oid, &cid, Some(&rec.id)) + .await + .unwrap(); + // A path-scoped rule so has_path_scoped_rule() is true (the tree-gate branch) + // without denying the "/" gate on the public repo. state .db .set_visibility_rule( &rec.id, - "src/**", + "/src/**", crate::db::VisibilityMode::B, - &["did:key:z6MkU3IpfsReaderAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &["did:key:z6MkF5IpfsTreeReaderAAAAAAAAAAAAAAAAAAAA".to_string()], &rec.owner_did, ) .await @@ -663,10 +1171,10 @@ mod tests { let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); - // Drive until the fake git's rev-list records its pid — the walk is now in the - // blocking pool and the request future is `.await`ing its join, holding the walk - // permit. Stop polling the instant the future completes (re-polling a completed - // oneshot panics). + // Drive until the fake git's rev-list records its pid — the TREE walk is now in + // the blocking pool and the request future is `.await`ing its join, holding the + // walk permit. Stop polling the instant the future completes (re-polling a + // completed oneshot panics). let mut walk_pid: Option = None; let mut early = None; for _ in 0..500 { @@ -696,28 +1204,28 @@ mod tests { } let _cleanup = ReapOnDrop(pid); - // Load-bearing: while the blocking walk runs, the slot is HELD and a replacement - // from a DIFFERENT source sheds 503 — proving the permit is retained across the - // spawn_blocking join, not freed by a tokio timeout. + // Load-bearing: while the blocking TREE walk runs, the slot is HELD and a + // replacement from a DIFFERENT source sheds 503 — proving the permit is + // retained across the spawn_blocking join, not freed by a tokio timeout. assert_eq!( sem.available_permits(), 0, - "the walk slot must be held while the spawn_blocking walk runs" + "the walk slot must be held while the spawn_blocking tree walk runs" ); let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, - "a replacement must shed 503 while the prior request's blocking walk still runs" + "a replacement must shed 503 while the prior request's blocking tree walk still runs" ); // Drop the in-flight request; the detached blocking walk keeps running (a - // spawn_blocking cannot be cancelled), but on the fix the permit is a handler - // local, so dropping the future releases it once the blocking join is abandoned. - // Either way, kill the sleeping child so the slot frees promptly and poll for - // recovery — the point already proven above is that the slot stayed held for the - // duration of the blocking work. + // spawn_blocking cannot be cancelled), but the permit is a handler local, so + // dropping the future releases it once the blocking join is abandoned. Either + // way, kill the sleeping child so the slot frees promptly and poll for + // recovery — the point already proven above is that the slot stayed held for + // the duration of the blocking work. drop(fut); unsafe { libc::kill(pid, libc::SIGKILL); @@ -736,155 +1244,6 @@ mod tests { ); } - /// Loop bound (cap N): one `/ipfs/{cid}` request against a CID present in many repos - /// must not serialize an unbounded number of full-history walks. With - /// `ipfs_max_repos_walked = 1` and TWO public, path-scoped repos both carrying the - /// blob at the requested CID, the handler walks only the FIRST candidate then stops - /// (the second is cut by the cap), so the fake git's `rev-list` (one per walk) runs - /// exactly once. MUTATION (RED): remove the `repos_walked >= cap` break and both - /// repos are walked (count 2). - #[cfg(unix)] - #[sqlx::test] - async fn get_by_cid_caps_repos_walked_per_request(pool: sqlx::PgPool) { - use std::process::Command; - - let tmp = tempfile::TempDir::new().unwrap(); - let walk_log = tmp.path().join("walks.log"); - // Fake git for the WALK: empty refs, `rev-parse` resolves, and each `rev-list` - // appends one line to a log (so the number of walks == the line count) and exits - // with EMPTY output (the allowed-set is empty, so every repo path-gates to a - // `continue` and the request 404s after walking). object_type uses the REAL git, - // so the seeded blob below must genuinely exist. - let body = format!( - "#!/bin/sh\n\ - case \"$1\" in\n\ - for-each-ref) : ;;\n\ - rev-parse) echo deadbeef ;;\n\ - rev-list) echo walk >> \"{}\" ;;\n\ - *) : ;;\n\ - esac\n\ - exit 0\n", - walk_log.display() - ); - let git_path = tmp.path().join("fakegit"); - std::fs::write(&git_path, &body).unwrap(); - { - use std::os::unix::fs::PermissionsExt; - let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); - perm.set_mode(0o755); - std::fs::set_permissions(&git_path, perm).unwrap(); - } - - let mut state = crate::test_support::test_state(pool.clone()).await; - let repos_dir = tmp.path().join("repos"); - std::fs::create_dir_all(&repos_dir).unwrap(); - state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); - state.git_bin = git_path.to_str().unwrap().to_string(); - state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; - // The bound under test: walk at most one candidate repo per request. - let mut cfg = (*state.config).clone(); - cfg.ipfs_max_repos_walked = 1; - state.config = Arc::new(cfg); - - // Seed TWO public repos, each with the SAME blob (same content -> same sha256 OID - // -> same CID) under a path-scoped rule, so both are walk candidates for one CID. - let run = |args: &[&str], cwd: &std::path::Path| { - let out = Command::new("git") - .args(args) - .current_dir(cwd) - .output() - .expect("git runs"); - assert!( - out.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&out.stderr) - ); - }; - let mut oid = String::new(); - for (i, name) in ["ipa", "ipb"].iter().enumerate() { - let owner = "z6ipfsN"; - state - .db - .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) - .await - .unwrap(); - let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); - let bare = state - .repo_store - .acquire(&rec.owner_did, &rec.name) - .await - .unwrap(); - let _ = std::fs::remove_dir_all(&bare); - std::fs::create_dir_all(&bare).unwrap(); - let work = tmp.path().join(format!("work{i}")); - std::fs::create_dir_all(work.join("src")).unwrap(); - // Identical content in both repos -> identical sha256 blob OID -> one CID. - std::fs::write(work.join("src/secret.txt"), b"loop bound proof\n").unwrap(); - run( - &["init", "-q", "--object-format=sha256", "-b", "main"], - &work, - ); - run(&["config", "user.email", "t@t"], &work); - run(&["config", "user.name", "t"], &work); - run(&["add", "src/secret.txt"], &work); - run(&["commit", "-q", "-m", "seed"], &work); - run( - &[ - "clone", - "--bare", - "-q", - work.to_str().unwrap(), - bare.to_str().unwrap(), - ], - tmp.path(), - ); - if oid.is_empty() { - let out = Command::new("git") - .args(["rev-parse", "HEAD:src/secret.txt"]) - .current_dir(&work) - .output() - .expect("git rev-parse runs"); - oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); - } - state - .db - .set_visibility_rule( - &rec.id, - "src/**", - crate::db::VisibilityMode::B, - &["did:key:z6MkU3IpfsReaderBBBBBBBBBBBBBBBBBBBBBBBB".to_string()], - &rec.owner_did, - ) - .await - .unwrap(); - } - let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); - let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) - .as_str() - .to_string(); - - let peer: SocketAddr = "203.0.113.90:5000".parse().unwrap(); - let mut req = Request::builder() - .method(Method::GET) - .uri(format!("/ipfs/{cid}")) - .body(Body::empty()) - .unwrap(); - req.extensions_mut().insert(ConnectInfo(peer)); - let resp = ipfs_router(state).oneshot(req).await.unwrap(); - // The empty allowed-set path-gates both repos to a `continue`, so a 404; the - // point is HOW MANY walks ran to get there. - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - let walks = std::fs::read_to_string(&walk_log) - .map(|s| s.lines().count()) - .unwrap_or(0); - assert_eq!( - walks, 1, - "with the per-request repo-walk cap at 1, only the first candidate repo is \ - walked (the second is cut by the cap), so exactly one walk runs; got {walks}" - ); - } - /// Route rate limit is WIRED (not a silent no-op): the production `build_router` /// attaches an `IpRateLimiter` extension to the `/ipfs/{cid}` route, so a per-IP /// flood is braked with 429. A bare `rate_limit_by_ip` layer with no extension does diff --git a/crates/gitlawb-node/src/api/mod.rs b/crates/gitlawb-node/src/api/mod.rs index 10d76daf..86594c03 100644 --- a/crates/gitlawb-node/src/api/mod.rs +++ b/crates/gitlawb-node/src/api/mod.rs @@ -200,9 +200,11 @@ mod authz_guard { (issues, "create_issue", "authorize_repo_read("), (bounties, "create_bounty", "authorize_repo_read("), (repos, "fork_repo", "authorize_repo_read("), - // get_by_cid gates each iterated repo row directly via visibility_check - // (KTD2a: it must NOT route through authorize_repo_read's fuzzy re-resolve). - (ipfs, "get_by_cid", "visibility_check("), + // get_by_cid resolves each candidate (provenance path + legacy scan) through + // the shared `gate_and_serve` (#173 round 2); the gate markers themselves are + // asserted below. This row proves the delegation is real — the gate is + // actually reached, not dead code. + (ipfs, "get_by_cid", "gate_and_serve("), // Bucket C — signer-self: the acting DID is matched/bound to auth.0 (tasks, "create_task", "did_matches("), (tasks, "claim_task", "did_matches("), @@ -236,6 +238,22 @@ mod authz_guard { "visibility::require_owner must use did_matches for DID-safe owner matching" ); + // The CID read surface (#173) enforces its gate inside the shared + // `gate_and_serve`, which BOTH the provenance path and the legacy scan call, so + // the markers must live there (the get_by_cid row above only proves delegation). + // The repo's own "/" visibility check (KTD2a — never authorize_repo_read's fuzzy + // re-resolve) and the quarantine hard-drop BEFORE visibility (INV-11) are both + // load-bearing: removing either re-opens a leak on the provenance path. + let gate_body = fn_body(ipfs, "gate_and_serve"); + assert!( + gate_body.contains("visibility_check("), + "gate_and_serve must gate the CID read surface via visibility_check (KTD2a)" + ); + assert!( + gate_body.contains("if quarantined"), + "gate_and_serve must hard-drop a quarantined repo before the visibility gate (INV-11)" + ); + for (src, func, marker) in rows { let body = fn_body(src, func); assert!( diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 9b05ece1..cb6ed541 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1473,6 +1473,7 @@ pub async fn git_receive_pack( &repo_path_clone, object_list_ipfs, &db_clone, + &repo_id, ) .await; if !pinned.is_empty() { @@ -1573,6 +1574,7 @@ pub async fn git_receive_pack( let pinata_upload_url = state.config.pinata_upload_url.clone(); let repo_path_clone = disk_path.clone(); let db_clone = state.db.clone(); + let repo_id = record.id.clone(); let http_client = Arc::clone(&state.http_client); let node_did_str = state.node_did.to_string(); let repo_slug = format!( @@ -1603,6 +1605,7 @@ pub async fn git_receive_pack( &repo_path_clone, object_list_pinata, &db_clone, + &repo_id, ) .await } else { diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 912108a0..c03eb2e5 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -516,6 +516,10 @@ mod tests { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), @@ -532,7 +536,6 @@ mod tests { git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), - ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 5324a4b9..6b0d394e 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -872,8 +872,63 @@ const MIGRATIONS: &[Migration] = &[ "CREATE UNIQUE INDEX IF NOT EXISTS idx_ref_certs_repo_ref ON ref_certificates(repo_id, ref_name)", ], }, + Migration { + version: 11, + name: "pinned_cids_cid_index", + stmts: &[ + // GET /ipfs/{cid} resolves an incoming CID -> git oid via pinned_cids.cid + // (#173); index it so the per-request lookup is not a table scan. This is + // a NEW versioned migration (not appended to the applied v1 bundle) so a + // node already past v1 actually gets the index. Non-unique on purpose: cid + // is a function of raw content, so a UNIQUE index could reject a legitimate + // record_pinned_cid insert, and colliding rows serve byte-identical content. + "CREATE INDEX IF NOT EXISTS idx_pinned_cids_cid ON pinned_cids(cid)", + ], + }, + Migration { + version: 12, + name: "pinned_cids_repo_provenance", + stmts: &[ + // Record the repository a pin came from so GET /ipfs/{cid} resolves a + // provenanced pin straight to its ONE source repo instead of scanning every + // repo (#173, jatmn round 2 — bounds the anonymous fan-out and removes the + // updated_at-ordering false-404). NEW versioned migration (never appended to + // the applied v1 pinned_cids table) so a node past v1 gets the column. + // Nullable: pins recorded before this migration have no provenance and fall + // back to the legacy repo scan; new pins carry repo_id and resolve to one + // repo. Indexed for the resolver's oid -> repo_id lookup. + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS repo_id TEXT", + "CREATE INDEX IF NOT EXISTS idx_pinned_cids_repo_id ON pinned_cids(repo_id)", + ], + }, + Migration { + version: 13, + name: "pin_repo_sources", + stmts: &[ + // F1 (#173, jatmn round 8): a shared object (a blob/tree/commit common to + // forks and mirrors) can be pinned from more than one repo. `pinned_cids` + // keeps only the FIRST pinner's `repo_id`, so a shared object first pinned + // from a private/quarantined repo 404s by CID even when a later PUBLIC repo + // also pinned it. Record EVERY pin-path source so `GET /ipfs/{cid}` can try + // each. NEW versioned migration (never appended to an applied block, INV-7). + // Bounded per object at insert time (MAX_PIN_SOURCES) so an adversary pushing + // one object from N repos cannot make resolution O(repos) (R2, INV-10). + "CREATE TABLE IF NOT EXISTS pin_repo_sources ( + sha256_hex TEXT NOT NULL, + repo_id TEXT NOT NULL, + PRIMARY KEY (sha256_hex, repo_id) + )", + "CREATE INDEX IF NOT EXISTS idx_pin_repo_sources_sha ON pin_repo_sources(sha256_hex)", + ], + }, ]; +/// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). +/// Bounds both the resolver's per-OID source loop and the `pin_repo_sources` growth, +/// so an adversary re-pushing one object from many repos cannot make resolution +/// O(repos) (R2, INV-10). +pub const MAX_PIN_SOURCES: i64 = 16; + // ── Repos ───────────────────────────────────────────────────────────────────── pub(crate) fn normalize_owner_key(did: &str) -> &str { @@ -1046,6 +1101,22 @@ impl Db { Ok(row.map(row_to_repo)) } + /// Fetch a repo by its stable `id`. Used by the `/ipfs/{cid}` provenance path, + /// which resolves a pin straight to its ONE source repo (#173) instead of + /// scanning `list_all_repos`. `id` is exact, so unlike `get_repo`'s fuzzy + /// owner/name match there is no mirror-vs-canonical disambiguation. + pub async fn get_repo_by_id(&self, id: &str) -> Result> { + let row = sqlx::query( + "SELECT id, name, owner_did, description, is_public, default_branch, + created_at, updated_at, disk_path, forked_from, machine_id + FROM repos WHERE id = $1 LIMIT 1", + ) + .bind(id) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(row_to_repo)) + } + #[allow(dead_code)] pub async fn list_repos(&self, owner_did: &str) -> Result> { let rows = sqlx::query( @@ -2166,20 +2237,148 @@ impl Db { Ok(row.get::("cnt") > 0) } - pub async fn record_pinned_cid(&self, sha256_hex: &str, cid: &str) -> Result<()> { + /// Every git oid a pinned CID maps to (`pinned_cids.cid` -> `sha256_hex`). + /// `GET /ipfs/{cid}` resolves the content-addressed CID a client sends back to + /// the object's git oid this way: a real pin CID digests the raw object + /// content, not the git oid, so the digest cannot be `git cat-file`d directly + /// (#173). The index is unique on the git oid but NON-unique on cid, so two + /// distinct oids can share one content-CID (a tree and a blob whose raw bytes + /// collide, or byte-identical content pinned under two oids). Returning every + /// candidate lets the handler try each rather than pick one arbitrarily and + /// false-404 when the chosen one is withheld or absent while another is + /// readable (#173). Empty when the CID was never pinned on this node. + pub async fn oids_for_cid(&self, cid: &str) -> Result> { + let rows = sqlx::query("SELECT sha256_hex FROM pinned_cids WHERE cid = $1") + .bind(cid) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| r.get::("sha256_hex")) + .collect()) + } + + /// Record a pinned object's CID and the repository it was pinned from + /// (`repo_id`, #173). On conflict the `COALESCE` backfills a NULL provenance + /// from a known source while keeping first-pinner-owns: an existing non-NULL + /// `repo_id` is never rewritten by a later push of the same oid, but a legacy + /// pin (or a pin recorded before provenance existed) whose `repo_id` is NULL + /// gets it filled the next time the object is re-pinned with a known source. + /// `cid`/`pinned_at` are left untouched on conflict. `repo_id` is `None` only + /// for a legacy pin with no known source; those fall back to the resolver's scan. + pub async fn record_pinned_cid( + &self, + sha256_hex: &str, + cid: &str, + repo_id: Option<&str>, + ) -> Result<()> { sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) - VALUES ($1, $2, $3) - ON CONFLICT(sha256_hex) DO NOTHING", + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT(sha256_hex) DO UPDATE SET + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", ) .bind(sha256_hex) .bind(cid) .bind(Utc::now().to_rfc3339()) + .bind(repo_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// The repository a pinned object was recorded from (`pinned_cids.repo_id`), + /// or `None` for a legacy pin (recorded before provenance existed) or an + /// unpinned oid. `GET /ipfs/{cid}` uses this to gate+serve the ONE source + /// repo instead of scanning every repo (#173). + pub async fn provenance_for_oid(&self, sha256_hex: &str) -> Result> { + let row = sqlx::query("SELECT repo_id FROM pinned_cids WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(row.and_then(|r| r.get::, _>("repo_id"))) + } + + /// Backfill the source repo on an already-pinned object whose provenance is + /// NULL (a legacy pin recorded before provenance existed, #173, jatmn). The + /// `AND repo_id IS NULL` guard keeps first-pinner-owns: an existing non-NULL + /// provenance is left untouched. Touches only `repo_id` and never re-pins the + /// object's bytes, so it is safe to call on the already-pinned skip path. + pub async fn backfill_pin_provenance(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + sqlx::query( + "UPDATE pinned_cids SET repo_id = $2 WHERE sha256_hex = $1 AND repo_id IS NULL", + ) + .bind(sha256_hex) + .bind(repo_id) .execute(&self.pool) .await?; Ok(()) } + /// Record a repository as a source for a pinned object (F1, #173 jatmn round 8), + /// bounded to about `MAX_PIN_SOURCES` distinct repos per object. The count guard + /// lives inside the INSERT (a single statement), which suppresses a re-push of the + /// SAME `(oid, repo)` via `ON CONFLICT DO NOTHING`. It does NOT hard-serialize + /// concurrent inserts of DIFFERENT repos for the same object: under Postgres READ + /// COMMITTED each concurrent writer's count subquery reads a snapshot that omits the + /// others' uncommitted rows, so N concurrent pushers can each see `count < cap` and + /// overshoot by up to N-1 rows. The overshoot is a small constant (bounded by + /// concurrent-pusher count, never O(repos)), and the RESOLVER read side + /// (`pin_sources_for_oid`) caps the ADDITIONAL sources at `MAX_PIN_SOURCES` (always + /// keeping the first-pinner), so the INV-10 bound on serve-time work holds at + /// `O(MAX_PIN_SOURCES + 1)` regardless of a table overshoot. + pub async fn record_pin_source(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + sqlx::query( + "INSERT INTO pin_repo_sources (sha256_hex, repo_id) + SELECT $1, $2 + WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo_id) + .bind(MAX_PIN_SOURCES) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Every source repository recorded for a pinned object (F1, #173 jatmn round 8): + /// the union of the first-pinner `pinned_cids.repo_id` and the `pin_repo_sources` + /// rows, deduped and ordered for a deterministic resolver walk. + /// + /// The first-pinner (a single row by `pinned_cids`' PK on `sha256_hex`) is ALWAYS + /// included; the `LIMIT MAX_PIN_SOURCES` caps only the ADDITIONAL `pin_repo_sources` + /// rows. This keeps the resolver's per-source work a bounded `O(MAX_PIN_SOURCES + 1)` + /// ceiling (INV-10) while never letting the cap evict the original source. A prior + /// version applied the `LIMIT` to the whole UNION with a lexicographic `ORDER BY`, + /// which let an attacker 404 a legacy public CID (first-pinner in `pinned_cids` but + /// not yet in `pin_repo_sources`) by pushing the same object from `MAX_PIN_SOURCES` + /// repos whose grindable ids sort before it, evicting the public source from the + /// window. Empty for a legacy pin with no known source (it falls back to the repo + /// scan) or an unpinned oid. + pub async fn pin_sources_for_oid(&self, sha256_hex: &str) -> Result> { + let rows = sqlx::query( + "SELECT repo_id FROM pinned_cids + WHERE sha256_hex = $1 AND repo_id IS NOT NULL + UNION + SELECT repo_id FROM ( + SELECT repo_id FROM pin_repo_sources + WHERE sha256_hex = $1 + ORDER BY repo_id + LIMIT $2 + ) capped + ORDER BY repo_id", + ) + .bind(sha256_hex) + .bind(MAX_PIN_SOURCES) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| r.get::("repo_id")) + .collect()) + } + pub async fn record_encrypted_blob( &self, repo_id: &str, @@ -2279,18 +2478,34 @@ impl Db { } /// Record the Pinata CID for a git object. - /// Inserts the row if it doesn't exist (objects pinned directly to Pinata - /// without a prior local IPFS pin get cid = pinata_cid). - pub async fn record_pinata_cid(&self, sha256_hex: &str, pinata_cid: &str) -> Result<()> { + /// + /// `raw_cid` is the locally-computed raw-content CID (`Cid::from_git_object_bytes`, + /// CIDv1/raw/sha2-256), the resolver key `GET /ipfs/{cid}` looks up; `pinata_cid` + /// is the provider CID Pinata returned (a dag-pb/UnixFS CID for gateway retrieval). + /// Inserts the row if it doesn't exist (an object pinned directly to Pinata with + /// no prior local IPFS pin gets `cid = raw_cid`, never the provider CID — a dag-pb + /// provider CID must never become an alias that serves raw bytes that do not hash + /// to it, #173). On conflict `cid` is left untouched: a prior local pin already + /// stored the correct raw CID, and the COALESCE backfills a NULL provenance from a + /// known source while keeping first-pinner-owns. + pub async fn record_pinata_cid( + &self, + sha256_hex: &str, + raw_cid: &str, + pinata_cid: &str, + repo_id: Option<&str>, + ) -> Result<()> { sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) - VALUES ($1, $2, $3, $4) - ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid", + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid, repo_id) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid, + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", ) .bind(sha256_hex) - .bind(pinata_cid) // fallback local cid if row is new + .bind(raw_cid) // resolver-key cid: locally-computed raw CID, never the provider CID .bind(Utc::now().to_rfc3339()) .bind(pinata_cid) + .bind(repo_id) .execute(&self.pool) .await?; Ok(()) @@ -5037,6 +5252,51 @@ mod ref_certificate_tests { ); } + /// INV-7: upgrade-path test — an existing node already past v1 must still get + /// the `pinned_cids.cid` index. It ships as its OWN v11 migration (not appended + /// to the applied v1 bundle), so dropping the index + its `schema_migrations` + /// row and re-running migrations must recreate it, exercising the real code + /// path rather than hand-copying the SQL. + #[sqlx::test] + async fn v11_pinned_cids_cid_index_applies_on_upgrade(pool: PgPool) { + async fn index_exists(pool: &PgPool) -> bool { + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM pg_indexes WHERE indexname = 'idx_pinned_cids_cid')", + ) + .fetch_one(pool) + .await + .unwrap() + } + + let db = Db::for_testing(pool.clone()); + db.run_migrations().await.unwrap(); + assert!( + index_exists(&pool).await, + "fresh migration chain creates the index" + ); + + // Simulate a node at v10 (pre-v11): drop the index and its migration record. + sqlx::query("DROP INDEX IF EXISTS idx_pinned_cids_cid") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 11") + .execute(&pool) + .await + .unwrap(); + assert!( + !index_exists(&pool).await, + "precondition: index and its migration record removed" + ); + + // Re-run migrations: v11 re-applies and recreates the index on the upgrade. + db.run_migrations().await.unwrap(); + assert!( + index_exists(&pool).await, + "v11 must recreate idx_pinned_cids_cid on an upgrading node" + ); + } + /// INV-7: upgrade-path test — seed a database at v9 with duplicate /// ref_certificates, then let the real v10 migration fire via /// run_migrations(). This exercises the migration code path rather than diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index a07235fa..371f0cf6 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -41,6 +41,9 @@ pub enum AppError { #[error("incomplete: {0}")] Incomplete(String), + #[error("search incomplete: {0}")] + SearchIncomplete(String), + #[error("git error: {0}")] Git(String), @@ -141,6 +144,15 @@ impl IntoResponse for AppError { AppError::Incomplete(msg) => { (StatusCode::UNPROCESSABLE_ENTITY, "incomplete", msg.clone()) } + // A bounded search that could not complete (the CID resolver hit its + // legacy-probe or walk ceiling), distinct from the 404 that asserts a + // definitive not-found: absence was NOT proven, so the caller should + // retry rather than treat it as gone (#173, F2). 503, retryable. + AppError::SearchIncomplete(msg) => ( + StatusCode::SERVICE_UNAVAILABLE, + "search_incomplete", + msg.clone(), + ), AppError::Git(msg) => (StatusCode::INTERNAL_SERVER_ERROR, "git_error", msg.clone()), // 504, distinct from the 500 git_error and from the read-gate's 404 / // the auth 401, so the client can tell a deadline from a failure. diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 229ee695..25f11246 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -290,6 +290,26 @@ pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> )) } +/// Object size in bytes (`git cat-file -s`) WITHOUT reading the content, so an +/// oversized object can be rejected before it is buffered into memory (#173, F6). +/// `None` if the object does not exist or the size is unparseable. +pub fn object_size(repo_path: &Path, sha256_hex: &str) -> Result> { + // allow-unbounded-git: cheap cat-file -s header read (no content), holds no served-git + // permit and cannot hang; exact twin of object_type above. Not an INV-22 lifecycle op. + let out = Command::new("git") + .args(["cat-file", "-s", sha256_hex]) + .current_dir(repo_path) + .output() + .context("failed to run git cat-file -s")?; + if !out.status.success() { + return Ok(None); + } + Ok(String::from_utf8_lossy(&out.stdout) + .trim() + .parse::() + .ok()) +} + /// Read an object's content if its type is already known. pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) -> Result> { let content_output = Command::new("git") diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 9f07020d..d463360c 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -648,6 +648,477 @@ pub fn allowed_blob_set_for_caller_bounded( Ok(allowed) } +/// The reachable-commit enumeration for the LENIENT walks (the `/ipfs/{cid}` tree +/// gate and the commit/tag reachability set): bounded `git rev-list --all [HEAD]` +/// under the caller's shared `deadline`, deliberately WITHOUT +/// `assert_all_refs_are_commits`. That guard fail-closes a repo's whole walk when +/// any ref peels to a non-commit (an annotated tag of a tree is pushable through +/// receive-pack), which would 404 every reachable tree/commit/tag CID here for a +/// legitimate reader. `rev-list --all` skips such refs cleanly, so the commit set +/// stays complete; an object reachable only via such a ref is simply excluded — +/// correctly fail-closed. Fails closed on a rev-list error. +/// +/// Safe ONLY for a caller whose output feeds a fail-closed allow-list where absence +/// = withhold: a tolerant walk there over-withholds, never leaks. NOT safe for a +/// serve/replication filter, where a missed reachable object under-withholds — +/// those go through `blob_paths`, which runs the guard first. +fn reachable_commit_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + // The HEAD probe is a bounded `git rev-parse --verify HEAD` (a clean exit means + // HEAD resolves), matching `blob_paths`. When HEAD does not resolve (unborn + // branch on an empty repo) `--all` alone yields nothing, which is correct. + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); + let mut rev_args = vec!["rev-list", "--all"]; + if head_resolves { + rev_args.push("HEAD"); + } + let out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + Ok(String::from_utf8_lossy(&out) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect()) +} + +/// Every `(oid, "/repo/relative/path", kind)` triple reachable from the given +/// `commits` — the shared ls-tree seam the tree walk filters (`kind == "tree"`). +/// One bounded `git ls-tree -rzt` per commit under the caller's shared `deadline`: +/// `-rzt` is byte-identical to `-rz` for blob records and additionally emits the +/// tree object for each directory at its own path. `kind` is git's object-type +/// string ("blob", "tree", or "commit" for a gitlink). The commit's ROOT tree is +/// not emitted by `ls-tree` (it lists entries *under* a tree); `tree_paths` adds +/// it. Triples are de-duplicated across commits and paths carry a leading "/" to +/// match the glob form of visibility rules ("/secret/**"). +/// +/// Fails closed: if any tree walk fails — or a path is not valid UTF-8 — it +/// returns an error so the caller aborts rather than producing a partial +/// (under-withheld) set. +fn object_paths( + repo_path: &Path, + git_bin: &str, + commits: &[String], + deadline: Instant, +) -> Result> { + let mut out: HashSet<(String, String, String)> = HashSet::new(); + for commit in commits { + let listing_out = run_bounded_git( + git_bin, + &["ls-tree", "-rzt", commit], + repo_path, + b"", + deadline, + )?; + // `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r` + // C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes + // "secret/caf\303\251.txt"), and that quoted literal would not match a + // visibility rule like "/secret/**", under-withholding the object. The TAB + // field separator survives `-z`, so the per-record parse is unchanged. + // + // Parse strictly: a lossy decode would replace an invalid byte in a denied + // path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string + // would no longer match its deny rule — the same under-withholding class, one + // layer down. Fail closed instead so the caller aborts rather than leaks. + let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { + anyhow::bail!( + "git ls-tree -rzt {commit} returned a non-UTF-8 path; \ + refusing to produce a partial (under-withheld) set" + ); + }; + for record in listing_stdout.split('\0') { + // " \t" + let Some((meta, path)) = record.split_once('\t') else { + continue; + }; + let mut parts = meta.split_whitespace(); + let _mode = parts.next(); + let kind = parts.next(); + let oid = parts.next(); + if let (Some(kind), Some(oid)) = (kind, oid) { + out.insert((oid.to_string(), format!("/{path}"), kind.to_string())); + } + } + } + Ok(out) +} + +/// Root tree oid of every reachable commit, at "/". `ls-tree` never emits a commit's +/// own root tree (it lists entries *under* a tree), so it is added explicitly here. +/// Resolved in ONE bounded `git log --no-walk --format=%T --stdin` pass over the +/// shared commit set — not a per-commit `rev-parse` — so a tree-set walk costs the +/// same subprocess order as the blob walk. The commit oids go on STDIN, not argv: a +/// long history has tens of thousands of reachable commits, and passing them all as +/// arguments overflows ARG_MAX so `git log` fails to spawn — which the caller treats +/// as a walk error and fail-closed 404s an authorized reader of a reachable/root +/// tree (#173 P2). `run_bounded_git` drains stdout concurrently with the stdin +/// write, so a large history cannot deadlock the pipes. A commit whose root tree git +/// cannot resolve fails the pass (bail), failing closed. +fn root_tree_pairs( + repo_path: &Path, + git_bin: &str, + commits: &[String], + deadline: Instant, +) -> Result> { + if commits.is_empty() { + return Ok(HashSet::new()); + } + let mut buf = String::with_capacity(commits.len() * 65); + for c in commits { + buf.push_str(c); + buf.push('\n'); + } + let out = run_bounded_git( + git_bin, + &["log", "--no-walk=unsorted", "--format=%T", "--stdin"], + repo_path, + buf.as_bytes(), + deadline, + )?; + let mut set = HashSet::new(); + for line in String::from_utf8_lossy(&out).lines() { + let oid = line.trim(); + if !oid.is_empty() { + set.insert((oid.to_string(), "/".to_string())); + } + } + Ok(set) +} + +/// Every `(tree_oid, "/path")` pair reachable in `repo_path`: the `kind == "tree"` +/// slice of [`object_paths`] (subtree trees at their directory paths) PLUS every +/// reachable commit's root tree at "/" (see [`root_tree_pairs`]). Computes the +/// reachable-commit set ONCE (leniently — see [`reachable_commit_oids`]; the tree +/// allowed-set feeds ONLY the `/ipfs/{cid}` tree gate, where absence = fail-closed +/// 404) and drives both the ls-tree walk and the root-tree pass from it, so the two +/// cannot diverge and neither re-enumerates. The tree analog of [`blob_paths`], +/// bounded by the same shared `deadline`. +fn tree_paths( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let commits = reachable_commit_oids(repo_path, git_bin, deadline)?; + let mut out: HashSet<(String, String)> = object_paths(repo_path, git_bin, &commits, deadline)? + .into_iter() + .filter(|(_, _, kind)| kind == "tree") + .map(|(oid, path, _)| (oid, path)) + .collect(); + out.extend(root_tree_pairs(repo_path, git_bin, &commits, deadline)?); + Ok(out) +} + +/// The OIDs from a `(oid, "/path")` listing that visibility ALLOWS `caller` at some +/// path — the shared inner loop of the blob and tree allowed-sets. An oid reachable +/// at an allowed path is kept even when also reachable at a denied one. +fn allowed_set_from_pairs<'a>( + pairs: impl IntoIterator, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> HashSet { + pairs + .into_iter() + .filter(|(_, path)| { + visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow + }) + .map(|(oid, _)| oid.clone()) + .collect() +} + +/// Reachable tree OIDs that visibility ALLOWS `caller` at some path — the tree +/// analog of [`allowed_blob_set_for_caller`]. `GET /ipfs/{cid}` gates tree objects +/// with this so the CID surface matches `get_tree`: a tree reachable only at a +/// withheld path is absent from the set and 404'd; the root tree ("/") and any tree +/// on the path to an allowed subtree are present. Fails closed on a +/// dangling/unreachable tree (never enumerated by the reachable walk, so never in +/// the set — the #126 geometry, for trees). A tree reachable at an allowed path is +/// included even when also reachable at a withheld one (its structure is visible to +/// this caller elsewhere). +#[cfg(test)] +pub fn allowed_tree_set_for_caller( + repo_path: &Path, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + allowed_tree_set_for_caller_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`allowed_tree_set_for_caller`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` tree gate. One deadline spans the whole walk (the HEAD +/// probe, rev-list, every per-commit ls-tree, and the root-tree pass), matching +/// `blob_paths`, so a slow or hung walk is bounded as a unit while the handler holds +/// its /ipfs walk permit (#174 F5). +pub fn allowed_tree_set_for_caller_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let deadline = Instant::now() + timeout; + Ok(allowed_set_from_pairs( + &tree_paths(repo_path, git_bin, deadline)?, + rules, + is_public, + owner_did, + caller, + )) +} + +/// Object bound for the annotated-tag reachability walk (#173, jatmn tag fan-out). +/// A path-scoped pinned-CID request drives this walk while holding one per-request +/// and one per-IP walk slot, so the total tag work must be finite regardless of how +/// many tag refs the repo has. 8192 is far past any real repo's annotated-tag count +/// (the Linux kernel has a few hundred), yet finite: a repo beyond it fails closed +/// (Err), matching this function's fail-closed-on-any-git-error contract, rather than +/// truncating silently (which would under-withhold a still-reachable tag object). +const MAX_TAG_OBJECTS: usize = 8192; + +/// Walk the annotated-tag chains rooted at `seeds`, inserting every tag object they +/// pass through into `set`. A tag whose target is itself a tag (tag-of-a-tag) +/// discovers the inner tag, which is walked in a later round. +/// +/// #173 (jatmn): the tag inspection is BATCHED, not one process per tag. Each round +/// feeds every not-yet-inspected tag oid to a SINGLE `git cat-file --batch` child on +/// stdin and reads back framed ` \n\n` records, so the +/// number of child processes is bounded by the tag-chain DEPTH (rounds), not the tag +/// COUNT. Oids go on stdin, never argv, so a large tag set cannot overflow ARG_MAX. +/// The child runs through [`run_bounded_git`], which drains stdout concurrently with +/// the stdin write (subsuming #173's F4 writer-thread drain — a round large enough to +/// fill both pipes cannot deadlock) and tears the child down at `deadline`, so a hung +/// cat-file cannot pin the caller's /ipfs walk permit (#174 F5). Total tag objects +/// inspected are capped at `max_tag_objects`; exceeding it is an error (fail closed), +/// not a silent truncation. Takes the bound as a parameter so a test can drive a tiny +/// value while the caller passes the real `MAX_TAG_OBJECTS`. +fn walk_tag_chain( + repo_path: &Path, + git_bin: &str, + seeds: Vec, + set: &mut HashSet, + max_tag_objects: usize, + deadline: Instant, +) -> Result<()> { + // Tag oids known but not yet inspected. Seeds may repeat / already be present; + // the `set.insert` gate below is what actually dedups and terminates cycles. + let mut pending: Vec = seeds; + let mut inspected: usize = 0; + + while !pending.is_empty() { + // Inspect only oids new to `set`; a re-seen oid was already walked. + let round: Vec = pending + .drain(..) + .filter(|oid| set.insert(oid.clone())) + .collect(); + if round.is_empty() { + break; + } + inspected += round.len(); + if inspected > max_tag_objects { + anyhow::bail!( + "annotated-tag walk exceeded the object bound ({max_tag_objects}); refusing to serve" + ); + } + + // One bounded child for the whole round: feed all oids on stdin, read the + // framed records from the returned stdout. + let mut buf = String::with_capacity(round.len() * 65); + for oid in &round { + buf.push_str(oid); + buf.push('\n'); + } + let stdout = run_bounded_git( + git_bin, + &["cat-file", "--batch"], + repo_path, + buf.as_bytes(), + deadline, + )?; + + // Parse one record per requested oid: ` \n\n`. + // A ` missing\n` record has no size/body and is anomalous here (every + // oid came from a ref tip or a prior tag body), so fail closed. + let mut i = 0usize; + for _ in 0..round.len() { + let hdr_end = stdout[i..] + .iter() + .position(|&b| b == b'\n') + .map(|p| i + p) + .context("git cat-file --batch: truncated record header")?; + let header = std::str::from_utf8(&stdout[i..hdr_end]) + .context("git cat-file --batch: non-utf8 record header")?; + i = hdr_end + 1; + let mut fields = header.split(' '); + let _oid = fields.next().unwrap_or(""); + let ty = fields.next().unwrap_or(""); + if ty == "missing" || fields.clone().next().is_none() { + anyhow::bail!("git cat-file --batch: object {header:?} missing or malformed"); + } + let size: usize = fields + .next() + .unwrap_or("") + .parse() + .context("git cat-file --batch: bad record size")?; + let body_end = i + .checked_add(size) + .filter(|&e| e <= stdout.len()) + .context("git cat-file --batch: truncated record body")?; + // Only a tag object can point at an inner tag; walk its header. + if ty == "tag" { + let body = std::str::from_utf8(&stdout[i..body_end]) + .context("git cat-file --batch: non-utf8 tag body")?; + let mut target = None; + let mut is_tag = false; + for line in body.lines() { + if let Some(oid) = line.strip_prefix("object ") { + target = Some(oid.trim().to_string()); + } else if line == "type tag" { + is_tag = true; + } else if line.is_empty() { + break; // end of header + } + } + if is_tag { + if let Some(t) = target { + pending.push(t); + } + } + } + // Skip body plus its trailing newline to the next record. + i = body_end + 1; + } + } + Ok(()) +} + +/// The reachable-commit/tag gate set for the `/ipfs/{cid}` resolver (#173, F2): +/// every reachable commit oid UNION every reachable annotated-tag OBJECT oid. A +/// DANGLING commit/tag (referenced by no ref, directly or via a tag chain) is in +/// neither part, so the resolver denies it under a path-scoped rule instead of +/// leaking its message; a reachable one still serves. +#[cfg(test)] +pub fn reachable_commit_tag_oids(repo_path: &Path) -> Result> { + reachable_commit_tag_oids_bounded(repo_path, "git", WALK_TIMEOUT) +} + +/// [`reachable_commit_tag_oids`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` commit/tag gate. One deadline spans the whole walk. +/// +/// Reachable commits come from bounded `git rev-list --all` (+ HEAD for the +/// detached case). Unlike the blob allowed-set, this does NOT run +/// `assert_all_refs_are_commits`: that guard fail-closes a repo's whole walk when +/// any ref peels to a non-commit (an annotated tag of a tree is pushable through +/// receive-pack), which would 404 every reachable commit/tag CID here for a +/// legitimate reader. The guard exists to stop blob/tree UNDER-withholding; it is +/// unnecessary for reachability, since a dangling object is absent from +/// `rev-list --all` and the ref walk below regardless of odd refs — so dropping it +/// recovers availability without admitting any dangling object (no leak). +/// +/// Reachable tag OBJECTS: `rev-list --all` dereferences annotated tags to commits, +/// so the tag objects are absent from it. Collect them by walking every ref tip and +/// peeling each tag's chain, so a nested tag-of-a-tag's INNER tag object (reachable +/// and pinnable, but not itself a ref tip) is included too. Fails closed on any git +/// error. +pub fn reachable_commit_tag_oids_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, +) -> Result> { + let deadline = Instant::now() + timeout; + // Reachable commits — no ref-commit assertion (see docstring). The HEAD probe + // doubles as the seed source for the tag-valued detached HEAD below: + // `rev-parse --verify HEAD` returns the tag oid UNPEELED when HEAD names a tag + // object. Failing to resolve HEAD (unborn/absent) is not fatal — there is + // simply no HEAD to walk or seed. + let head_oid: Option = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .ok() + .map(|out| String::from_utf8_lossy(&out).trim().to_string()) + .filter(|s| !s.is_empty()); + let mut rev_args = vec!["rev-list", "--all"]; + if head_oid.is_some() { + rev_args.push("HEAD"); + } + let rev = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let mut set: HashSet = String::from_utf8_lossy(&rev) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect(); + + // Ref tips that are annotated tag objects seed the tag-chain walk. + let refs = run_bounded_git( + git_bin, + &["for-each-ref", "--format=%(objectname) %(objecttype)"], + repo_path, + b"", + deadline, + )?; + let mut worklist: Vec = Vec::new(); + for line in String::from_utf8_lossy(&refs).lines() { + let mut it = line.split_whitespace(); + if let (Some(oid), Some("tag")) = (it.next(), it.next()) { + worklist.push(oid.to_string()); + } + } + // A detached/direct HEAD may name an annotated tag object with no ref at that tag + // (#173 review, finding 3): `rev-list --all HEAD` above peels it to its commit and + // `for-each-ref` has no tag row, so the tag OBJECT would be omitted and its pinned + // CID would 404 for an authorized reader. Seed a tag-valued HEAD into the tag-chain + // walk; a `commit` HEAD adds nothing. A cat-file failure here only skips the seed + // (over-withholds that one tag — fail-closed), matching the original's tolerance. + if let Some(head_oid) = head_oid { + if let Ok(ty) = run_bounded_git( + git_bin, + &["cat-file", "-t", &head_oid], + repo_path, + b"", + deadline, + ) { + if String::from_utf8_lossy(&ty).trim() == "tag" { + worklist.push(head_oid); + } + } + } + // Peel every tag object's chain into `set`, adding each tag object it passes + // through. Bounded and batched (#173, jatmn tag fan-out): see `walk_tag_chain`. + walk_tag_chain( + repo_path, + git_bin, + worklist, + &mut set, + MAX_TAG_OBJECTS, + deadline, + )?; + Ok(set) +} + /// Objects safe to replicate, failing closed on blobs (#99). A candidate /// replicates iff it is NOT a blob (`all_blob_oids` — commits and trees are /// structural, never content-withheld) OR it is in `allowed_blobs` (reachable @@ -1068,6 +1539,614 @@ esac\n"; (td, bare, secret, public) } + /// #173 (jatmn round 8, F4 — load-bearing): a repo with enough annotated tags that + /// one `cat-file --batch` round fills BOTH pipes (stdin > 64 KiB of oids while the + /// child blocks on a full stdout) must not deadlock. The old order wrote the whole + /// round to stdin before draining stdout and hung indefinitely, stranding a blocking- + /// pool thread; `run_bounded_git`'s concurrent writer/drain completes. Driven with a + /// completion timeout: GREEN finishes in well under a second, RED (old order) hangs + /// and the recv_timeout fires. ~3000 tags is well past the ~2030-oid deadlock + /// threshold (41 bytes/oid, 64 KiB pipes) and under MAX_TAG_OBJECTS (8192). + /// Bulk-created via one fast-import stream so the fixture cost is one git process, + /// not 3000 `git tag -a` spawns. + #[test] + fn walk_tag_chain_large_batch_does_not_deadlock() { + use std::io::Write; + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + std::fs::create_dir_all(&work).unwrap(); + std::fs::write(work.join("f.txt"), b"x\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "init"], &work); + let head = { + let out = Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Bulk-create ~3000 annotated tags via one fast-import stream. + const N: usize = 3000; + let mut stream = String::new(); + for i in 0..N { + let msg = format!("annotated tag {i}\n"); + stream.push_str(&format!("tag t{i}\n")); + stream.push_str(&format!("from {head}\n")); + stream.push_str("tagger t 1700000000 +0000\n"); + stream.push_str(&format!("data {}\n", msg.len())); + stream.push_str(&msg); + } + let mut fi = Command::new("git") + .args(["fast-import", "--quiet"]) + .current_dir(&work) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + fi.stdin + .take() + .unwrap() + .write_all(stream.as_bytes()) + .unwrap(); + assert!(fi.wait().unwrap().success(), "fast-import failed"); + + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + + // Drive the walk on a worker thread with a completion timeout. The old + // write-all-before-drain order hangs here; the fix completes near-instantly. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(reachable_commit_tag_oids(&bare).map(|s| s.len())); + }); + match rx.recv_timeout(std::time::Duration::from_secs(20)) { + Ok(Ok(n)) => assert!( + n >= N, + "the walk must resolve every annotated tag object (got {n}, expected >= {N})" + ), + Ok(Err(e)) => panic!("walk errored: {e}"), + Err(_) => panic!("walk_tag_chain deadlocked on a large tag batch (F4 regression)"), + } + } + + /// #173 review (finding 3): an annotated tag reachable ONLY through a tag-valued + /// detached HEAD (raw HEAD naming a tag object, with no ref at that tag) must still + /// enter `reachable_commit_tag_oids`. `rev-list --all HEAD` peels such a HEAD to its + /// commit and `for-each-ref` has no tag row, so without a HEAD tag-seed the tag + /// OBJECT is omitted and its pinned CID would 404 for an authorized reader. RED + /// before the HEAD tag-seed (the tag oid is absent); GREEN after. + #[test] + fn reachable_commit_tag_oids_includes_tag_valued_detached_head() { + use std::io::Write; + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + std::fs::create_dir_all(&work).unwrap(); + std::fs::write(work.join("a.txt"), b"hi\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "seed"], &work); + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + let commit = run(&["rev-parse", "HEAD"], &bare); + + // An annotated tag OBJECT in the bare ODB, with NO ref pointing at it. + let tag_body = format!( + "object {commit}\ntype commit\ntag htag\ntagger t 0 +0000\n\nHEAD-only tag\n" + ); + let tag_oid = { + let mut child = Command::new("git") + .args(["hash-object", "-t", "tag", "-w", "--stdin"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .as_mut() + .unwrap() + .write_all(tag_body.as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + assert_eq!(run(&["cat-file", "-t", &tag_oid], &bare), "tag"); + // Raw-write HEAD directly to the tag object (the only way this state arises; + // update-ref / checkout both refuse a non-commit HEAD). + std::fs::write(bare.join("HEAD"), format!("{tag_oid}\n")).unwrap(); + + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!( + set.contains(&tag_oid), + "a tag reachable only via a tag-valued detached HEAD must be in the reachable set" + ); + assert!( + set.contains(&commit), + "the commit the HEAD tag peels to stays reachable (no regression)" + ); + } + + /// #173: `reachable_commit_tag_oids` on an empty repo (unborn HEAD) must return an + /// empty set, not error — exercising the `rev-parse HEAD` fail branch of the + /// detached-HEAD tag seed (there is simply no HEAD to seed). + #[test] + fn reachable_commit_tag_oids_handles_unborn_head() { + let td = TempDir::new().unwrap(); + let bare = td.path().join("empty.git"); + let ok = Command::new("git") + .args(["init", "-q", "--bare", bare.to_str().unwrap()]) + .status() + .unwrap() + .success(); + assert!(ok, "git init --bare failed"); + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!( + set.is_empty(), + "an empty repo (unborn HEAD) yields an empty reachable set with no error" + ); + } + + #[test] + fn object_paths_emits_trees_and_blob_paths_is_the_blob_slice() { + let (_td, bare, secret_oid, public_oid) = fixture(); + let deadline = Instant::now() + WALK_TIMEOUT; + // The lenient enumeration; on this clean fixture it matches the strict one. + let commits = reachable_commit_oids(&bare, "git", deadline).unwrap(); + let objs = object_paths(&bare, "git", &commits, deadline).unwrap(); + + // Blob records survive the `-rzt` change, at their paths (unchanged). + assert!(objs.contains(&(secret_oid.clone(), "/secret/b.txt".into(), "blob".into()))); + assert!(objs.contains(&(public_oid.clone(), "/public/a.txt".into(), "blob".into()))); + + // The #135 addition: subtree tree objects at their directory paths. + assert!( + objs.iter().any(|(_, p, k)| k == "tree" && p == "/secret"), + "the /secret subtree tree must be emitted at its dir path" + ); + assert!( + objs.iter().any(|(_, p, k)| k == "tree" && p == "/public"), + "the /public subtree tree must be emitted at its dir path" + ); + + // blob_paths must equal the blob slice of object_paths exactly — compared as + // SETS (both walks dedup via HashSet; the collected order is nondeterministic). + let bp: HashSet<(String, String)> = blob_paths(&bare, "git", WALK_TIMEOUT) + .unwrap() + .into_iter() + .collect(); + let bp_from_obj: HashSet<(String, String)> = objs + .iter() + .filter(|(_, _, k)| k == "blob") + .map(|(o, p, _)| (o.clone(), p.clone())) + .collect(); + assert_eq!( + bp, bp_from_obj, + "blob_paths output must be byte-identical to object_paths' blob slice" + ); + } + + #[test] + fn allowed_tree_set_gates_withheld_subtree_tree() { + let (_td, bare, _s, _p) = fixture(); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let secret_tree = oid("HEAD:secret"); + let public_tree = oid("HEAD:public"); + let root_tree = oid("HEAD^{tree}"); + let reader = "did:key:z6MkReader"; + let rules = [rule("/secret/**", &[reader])]; + + // anon: the withheld /secret tree is excluded; root ("/") and /public are in. + let anon = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, None).unwrap(); + assert!( + !anon.contains(&secret_tree), + "withheld /secret subtree tree excluded for anon" + ); + assert!(anon.contains(&root_tree), "root tree included (path /)"); + assert!(anon.contains(&public_tree), "/public subtree tree included"); + + // listed reader: sees the /secret tree (caller-aware, not a blanket deny). + let rd = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, Some(reader)).unwrap(); + assert!( + rd.contains(&secret_tree), + "listed reader sees the /secret tree" + ); + + // owner: sees every reachable tree. + let ow = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, Some(OWNER)).unwrap(); + assert!( + ow.contains(&secret_tree) && ow.contains(&public_tree) && ow.contains(&root_tree), + "owner sees all reachable trees" + ); + } + + #[test] + fn allowed_tree_set_excludes_dangling_tree() { + use std::io::Write; + let (_td, bare, secret_oid, _p) = fixture(); + // A DANGLING tree: written to the ODB but referenced by no commit. Uses a + // UNIQUE entry name so its oid is content-distinct from every reachable tree + // (a content-identical tree would dedup to a reachable oid — that is T2, not + // danglingness). The reachable-only walk never enumerates it -> fail closed. + let mut child = Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {secret_oid}\tdangling-only-unreferenced.txt" + ) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "git mktree"); + let dangling = String::from_utf8_lossy(&out.stdout).trim().to_string(); + + let rules = [rule("/secret/**", &[])]; + for caller in [None, Some(OWNER)] { + let set = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, caller).unwrap(); + assert!( + !set.contains(&dangling), + "dangling tree must never be in the reachable allowed-set (caller={caller:?})" + ); + } + } + + #[test] + fn allowed_tree_set_includes_tree_shared_across_allowed_and_denied_paths() { + // T2 (content-dedup): the SAME tree oid reachable at both an allowed and a + // withheld path is INCLUDED for anon (allowed-wins) — its structure is + // visible to the caller at the allowed path. Mirrors the blob analog + // `same_blob_at_allowed_and_denied_path_is_not_withheld`. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(work.join("pub/sub")).unwrap(); + std::fs::create_dir_all(work.join("sec/sub")).unwrap(); + std::fs::write(work.join("pub/sub/f.txt"), b"same bytes\n").unwrap(); + std::fs::write(work.join("sec/sub/f.txt"), b"same bytes\n").unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + run(&["add", "."]); + run(&["commit", "-qm", "seed"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let pub_sub = oid("HEAD:pub/sub"); + let sec_sub = oid("HEAD:sec/sub"); + assert_eq!(pub_sub, sec_sub, "identical content dedups to one tree oid"); + + // Withhold /sec from anon; the shared oid is still reachable at /pub/sub. + let rules = [rule("/sec/**", &[])]; + let anon = allowed_tree_set_for_caller(&work, &rules, true, OWNER, None).unwrap(); + assert!( + anon.contains(&pub_sub), + "a tree reachable at an allowed path is included even when also at a withheld path" + ); + } + + #[test] + fn allowed_tree_set_includes_root_trees_of_all_reachable_commits() { + // The batched root-tree pass (root_tree_pairs) must return EVERY reachable + // commit's root tree, not just HEAD's — two commits with distinct root trees + // both land in the set. Guards the git-log-over-N-commits root derivation. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + std::fs::write(work.join("a.txt"), b"one\n").unwrap(); + run(&["add", "."]); + run(&["commit", "-qm", "c1"]); + let root1 = oid("HEAD^{tree}"); + std::fs::write(work.join("b.txt"), b"two\n").unwrap(); + run(&["add", "."]); + run(&["commit", "-qm", "c2"]); + let root2 = oid("HEAD^{tree}"); + assert_ne!(root1, root2, "the two commits have distinct root trees"); + + // Public repo, no rules: every reachable tree is allowed for anon. + let set = allowed_tree_set_for_caller(&work, &[], true, OWNER, None).unwrap(); + assert!( + set.contains(&root1) && set.contains(&root2), + "root trees of BOTH reachable commits are in the set (batched root pass)" + ); + } + + #[test] + fn root_tree_pairs_returns_every_root_tree_at_scale() { + // Parity + liveness at scale for root_tree_pairs (#173 P2): feed every + // reachable commit oid to `git log --format=%T --stdin` and collect each + // commit's root tree. With N commits that is ~N*41 bytes of oids in and + // ~N*41 bytes of %T out — past the ~64 KiB pipe buffer in both directions — + // so this exercises the large-bidirectional-IO path the 2-commit test above + // cannot, and asserts parity: every distinct root tree comes back. + // + // NOTE: this is NOT a deadlock guard. `git log --stdin` reads its whole + // revision list to EOF before emitting any %T, so the naive "write all of + // stdin, then drain stdout" form does not deadlock at any scale for this + // invocation. `run_bounded_git`'s concurrent writer/drain is cheap defensive + // isolation, not load-bearing, and this test does not claim otherwise. The + // 30s watchdog is a general liveness bound so a future regression that + // genuinely hangs fails fast here rather than stalling the suite. + const N: usize = 2500; + let td = TempDir::new().unwrap(); + let bare = td.path().join("many.git"); + assert!(Command::new("git") + .args(["init", "-q", "--bare", bare.to_str().unwrap()]) + .status() + .unwrap() + .success()); + + // fast-import a linear chain of N commits, each adding a distinct file so + // every root tree is distinct (dedup cannot shrink the output). One + // subprocess, ~1s — far cheaper than N `git commit` spawns. + let mut stream = String::new(); + for i in 0..N { + let (b, cm) = (2 * i + 1, 2 * i + 2); + let content = format!("v{i}"); + let msg = format!("c{i}"); + stream.push_str(&format!( + "blob\nmark :{b}\ndata {}\n{content}\n", + content.len() + )); + stream.push_str(&format!( + "commit refs/heads/main\nmark :{cm}\ncommitter t 0 +0000\ndata {}\n{msg}\n", + msg.len() + )); + if i > 0 { + stream.push_str(&format!("from :{}\n", 2 * (i - 1) + 2)); + } + stream.push_str(&format!("M 100644 :{b} f{i}\n\n")); + } + let mut fi = Command::new("git") + .args(["fast-import", "--quiet"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + { + use std::io::Write; + fi.stdin + .take() + .unwrap() + .write_all(stream.as_bytes()) + .unwrap(); + } + assert!(fi.wait().unwrap().success(), "fast-import failed"); + + let commits = reachable_commit_oids(&bare, "git", Instant::now() + WALK_TIMEOUT).unwrap(); + assert_eq!(commits.len(), N, "all {N} commits reachable"); + + // Call root_tree_pairs directly (private, same module) under a liveness + // watchdog, then assert it returned every distinct root tree. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send( + root_tree_pairs(&bare, "git", &commits, Instant::now() + WALK_TIMEOUT) + .map(|s| s.len()), + ); + }); + match rx.recv_timeout(std::time::Duration::from_secs(30)) { + Ok(Ok(len)) => assert_eq!(len, N, "every distinct root tree returned"), + Ok(Err(e)) => panic!("root_tree_pairs errored: {e}"), + Err(_) => panic!("root_tree_pairs did not return within 30s"), + } + } + + /// #173 (jatmn tag fan-out): the batched `git cat-file --batch` tag walk must + /// return the SAME reachable set as the old per-tag `cat-file tag` loop — every + /// commit, the outer tag object, AND the inner tag object of a tag-of-a-tag chain + /// (the inner tag is reachable but is not itself a ref tip, so it is only found by + /// peeling the outer tag's target). Behavior-preservation proof for the rewrite. + #[test] + fn reachable_commit_tag_oids_includes_nested_tag_objects() { + let (_td, bare, _secret, _public) = fixture(); + let run = |args: &[&str]| -> String { + let out = Command::new("git") + .args(args) + .current_dir(&bare) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + // v1 -> commit, v2 -> v1 (tag-of-a-tag), plus a couple of sibling tags so the + // round batches more than one oid. Capture v1's oid, then DELETE the v1 ref so + // the inner tag object survives in the ODB but is NOT a ref tip: it is then + // reachable ONLY by peeling v2's target chain. That makes the peel load-bearing + // (breaking the inner-tag enqueue drops v1 from the set), unlike leaving v1 as + // its own ref where `for-each-ref` would seed it directly. + run(&["tag", "-a", "-m", "inner", "v1", "HEAD"]); + run(&["tag", "-a", "-m", "outer", "v2", "v1"]); + run(&["tag", "-a", "-m", "s1", "s1", "HEAD"]); + run(&["tag", "-a", "-m", "s2", "s2", "HEAD"]); + let commit = run(&["rev-parse", "HEAD"]); + let v1 = run(&["rev-parse", "v1"]); + let v2 = run(&["rev-parse", "v2"]); + let s1 = run(&["rev-parse", "s1"]); + let s2 = run(&["rev-parse", "s2"]); + run(&["tag", "-d", "v1"]); + + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!(set.contains(&commit), "the commit must be reachable"); + assert!( + set.contains(&v2), + "the outer tag object (ref tip) must be present" + ); + assert!( + set.contains(&v1), + "the INNER tag object of a tag-of-a-tag must be present (peeled from v2, no ref)" + ); + assert!(set.contains(&s1), "sibling tag s1 must be present"); + assert!(set.contains(&s2), "sibling tag s2 must be present"); + } + + /// #173 (jatmn tag fan-out): the object bound is load-bearing. A repo whose tag + /// count exceeds the bound must FAIL CLOSED (Err), not return a truncated set that + /// would under-withhold a still-reachable tag. Drives `walk_tag_chain` with a tiny + /// injected bound (the public fn uses the real `MAX_TAG_OBJECTS`); with the bound + /// check removed this would collect all tags and return Ok. + #[test] + fn walk_tag_chain_fails_closed_over_object_bound() { + let (_td, bare, _secret, _public) = fixture(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&bare) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + let mut seeds = Vec::new(); + for n in 0..5 { + let name = format!("t{n}"); + run(&["tag", "-a", "-m", &name, &name, "HEAD"]); + let oid = Command::new("git") + .args(["rev-parse", &name]) + .current_dir(&bare) + .output() + .unwrap(); + seeds.push(String::from_utf8_lossy(&oid.stdout).trim().to_string()); + } + + // Within a generous bound: the walk succeeds and collects the tags. + let mut ok_set = HashSet::new(); + walk_tag_chain( + &bare, + "git", + seeds.clone(), + &mut ok_set, + 8192, + Instant::now() + WALK_TIMEOUT, + ) + .unwrap(); + assert!( + seeds.iter().all(|s| ok_set.contains(s)), + "all 5 tags collected under a generous bound" + ); + + // Under a bound of 2 with 5 tags: fail closed (Err), not a partial set. + let mut small_set = HashSet::new(); + let result = walk_tag_chain( + &bare, + "git", + seeds, + &mut small_set, + 2, + Instant::now() + WALK_TIMEOUT, + ); + assert!( + result.is_err(), + "a tag count exceeding the object bound must fail closed (Err), not truncate" + ); + } + #[test] fn anonymous_caller_withholds_only_private_blob() { let (_td, bare, secret_oid, public_oid) = fixture(); diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 3b346190..a4ed8e4d 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -91,7 +91,8 @@ pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { /// `..._fail_closed` filter on the full-scan path before calling, so this /// function never sees a withheld blob. `repo_path` is still needed to read each /// object's bytes. The twin in `pinata.rs` mirrors this shape — change both in -/// lockstep. +/// lockstep. `repo_id` records the pin's provenance so `GET /ipfs/{cid}` resolves +/// straight to this repo instead of scanning every repo (#173). /// /// Returns a list of `(sha256_hex, cid)` pairs for objects pinned this call. pub async fn pin_new_objects( @@ -99,6 +100,7 @@ pub async fn pin_new_objects( repo_path: &std::path::Path, object_list: Vec, db: &crate::db::Db, + repo_id: &str, ) -> Vec<(String, String)> { if ipfs_api.is_empty() { return vec![]; @@ -107,9 +109,36 @@ pub async fn pin_new_objects( let mut pinned = Vec::new(); for sha in object_list { - // Skip if already pinned + // Skip if already pinned — but first backfill provenance if the existing + // pin has none. A legacy pin (recorded before repo_id existed, #173, jatmn) + // is skipped here before record_pinned_cid ever runs, so its NULL provenance + // would never resolve to one repo and known CIDs keep hitting the scan. The + // backfill only sets repo_id (AND repo_id IS NULL guard preserves + // first-pinner-owns) and never re-pins the bytes — the object is already on IPFS. match db.is_pinned(&sha).await { - Ok(true) => continue, + Ok(true) => { + match db.provenance_for_oid(&sha).await { + Ok(None) => { + if let Err(e) = db.backfill_pin_provenance(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to backfill pin provenance"); + } + } + Ok(Some(_)) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "DB error reading pin provenance"); + } + } + // F1 (#173 round 8): record this repo as an ADDITIONAL source for the + // already-pinned object. This is the load-bearing skip-branch insert — + // a later repo pushing a shared object hits this path (already pinned), + // and without it `GET /ipfs/{cid}` only ever knows the first pinner, so a + // shared object first pinned from a private/quarantined repo 404s even + // when this repo would serve it. Bounded per object (MAX_PIN_SOURCES). + if let Err(e) = db.record_pin_source(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } + continue; + } Ok(false) => {} Err(e) => { tracing::warn!(sha = %sha, err = %e, "DB error checking pinned status"); @@ -130,9 +159,14 @@ pub async fn pin_new_objects( // Pin to IPFS match pin_git_object(ipfs_api, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinned_cid(&sha, &cid).await { + if let Err(e) = db.record_pinned_cid(&sha, &cid, Some(repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); } + // F1 (#173 round 8): also record the first pinner in pin_repo_sources so + // every source (first and subsequent) is tried uniformly by the resolver. + if let Err(e) = db.record_pin_source(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } pinned.push((sha, cid)); } Ok(_) => {} diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index ffac65a2..37190c76 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -374,6 +374,9 @@ async fn main() -> Result<()> { rate_limiter, create_ip_rate_limiter, push_rate_limiter, + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust, sync_trigger_rate_limiter, peer_write_rate_limiter, @@ -465,22 +468,16 @@ async fn main() -> Result<()> { // Periodic cleanup of expired rate limit entries + consumed-proof ledger { - let rl = state.rate_limiter.clone(); - let create_ip_rl = state.create_ip_rate_limiter.clone(); - let push_rl = state.push_rate_limiter.clone(); - let sync_trigger_rl = state.sync_trigger_rate_limiter.clone(); - let peer_write_rl = state.peer_write_rate_limiter.clone(); + let cleanup_state = state.clone(); let db = state.db.clone(); let mut shutdown_rx = state.subscribe_shutdown(); tokio::spawn(async move { loop { tokio::select! { _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { - rl.cleanup().await; - create_ip_rl.cleanup().await; - push_rl.cleanup().await; - sync_trigger_rl.cleanup().await; - peer_write_rl.cleanup().await; + // Sweep every per-IP/DID limiter (incl. the ipfs walk brake) + // so bounded maps shed stale keys instead of sitting at cap. + cleanup_state.sweep_rate_limiters().await; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs() as i64) diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 6c9c0bff..31843bbc 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -74,8 +74,10 @@ pub async fn pin_object( /// `..._fail_closed` filter on the full-scan path before calling. `repo_path` is /// still needed to read each object's bytes. The twin in `ipfs_pin.rs` mirrors /// this shape — change both in lockstep. Objects already recorded with a -/// `pinata_cid` are skipped. Returns `(sha_hex, cid)` pairs for each newly -/// pinned object. +/// `pinata_cid` are skipped. `repo_id` records the pin's provenance (#173). +/// Returns `(sha_hex, provider_cid)` pairs for each newly pinned object: the +/// provider CID is the Pinata gateway CID (used for branch→CID recording and +/// ref-update gossip), NOT the raw resolver-key CID stored in `pinned_cids.cid`. pub async fn pin_new_objects( client: &reqwest::Client, upload_url: &str, @@ -83,6 +85,7 @@ pub async fn pin_new_objects( repo_path: &std::path::Path, object_list: Vec, db: &crate::db::Db, + repo_id: &str, ) -> Vec<(String, String)> { if jwt.is_empty() { return vec![]; @@ -92,7 +95,15 @@ pub async fn pin_new_objects( for sha in object_list { match db.has_pinata_cid(&sha).await { - Ok(true) => continue, + Ok(true) => { + // F1 (#173 round 8): record this repo as an additional source for the + // already-pinned object (mirrors the ipfs_pin skip-branch insert) so the + // resolver can serve a shared object from any pin-path source. + if let Err(e) = db.record_pin_source(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } + continue; + } Ok(false) => {} Err(e) => { tracing::warn!(sha = %sha, err = %e, "DB error checking pinata_cid"); @@ -111,9 +122,21 @@ pub async fn pin_new_objects( match pin_object(client, upload_url, jwt, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinata_cid(&sha, &cid).await { + // The resolver key (`pinned_cids.cid`) must be the locally-computed + // raw-content CID, never the provider CID: Pinata wraps the bytes in + // dag-pb/UnixFS, so its returned CID does not hash the raw content and + // must not become an alias `/ipfs/{cid}` serves raw git bytes for (#173). + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); + if let Err(e) = db + .record_pinata_cid(&sha, &raw_cid, &cid, Some(repo_id)) + .await + { tracing::warn!(sha = %sha, err = %e, "failed to record pinata_cid in DB"); } + // F1 (#173 round 8): also record the first pinner in pin_repo_sources. + if let Err(e) = db.record_pin_source(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } pinned.push((sha, cid)); } Ok(_) => {} diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index 22281691..14c11a9c 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -121,6 +121,28 @@ impl RateLimiter { true } + /// Non-consuming check: is this key ALREADY at its limit for the current window? + /// Unlike [`check`], it records nothing and never inserts a new key — used to shed + /// expensive preparatory work (e.g. the `/ipfs/{cid}` legacy scan's O(repos) DB + /// preload) BEFORE it runs, without perturbing the per-unit budget the consuming + /// `check` maintains (#173, F3). An unknown key or a disabled limiter is not + /// throttled. Prunes the key's expired timestamps as a side effect (keeps state + /// tidy) but adds none, so it cannot itself fill or grow the map. + pub(crate) async fn is_throttled(&self, key: &str) -> bool { + if self.max_requests == 0 { + return false; + } + let now = Instant::now(); + let mut state = self.state.lock().await; + if let Some(window) = state.get_mut(key) { + window + .timestamps + .retain(|t| now.duration_since(*t) < self.window); + return window.timestamps.len() >= self.max_requests; + } + false + } + pub async fn cleanup(&self) { let now = Instant::now(); let mut state = self.state.lock().await; @@ -130,6 +152,13 @@ impl RateLimiter { !w.timestamps.is_empty() }); } + + /// Number of distinct keys currently tracked. Test-only introspection so a + /// cross-module test can assert that a sweep actually evicted expired entries. + #[cfg(test)] + pub(crate) async fn tracked_keys(&self) -> usize { + self.state.lock().await.len() + } } /// A bounded per-caller CONCURRENCY limiter — distinct from [`RateLimiter`], which diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index deccf4c7..10dbbf32 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -65,6 +65,29 @@ pub struct AppState { /// brake a push flood from a DID farm (one throwaway DID per repo), so the /// push path throttles on the resolved client IP instead. pub push_rate_limiter: RateLimiter, + /// Per-client-IP rate limiter for the `GET /ipfs/{cid}` full-history walk. + /// The route is anonymous and a valid tree CID (exposed by the public pins + /// index) makes each repeat request pay a fresh allowed-set walk (rev-list + + /// ls-tree per commit), memoized only per request — unbounded amplification + /// (INV-10). Braking the walk on the non-farmable source IP caps that cost + /// without touching cheap non-walk fetches. Keyed by `push_limiter_trust`. + pub ipfs_rate_limiter: RateLimiter, + /// Per-request ceiling on full-history reachability walks the CID resolver + /// may spawn (default `api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST`). A field, + /// not a bare const, so tests can shrink it to exercise the cap cheaply; + /// production keeps the const default. + pub ipfs_max_history_walks: u32, + /// Per-request ceiling on legacy (NULL-provenance) repo probes in the CID + /// resolver's scan fallback (default `api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST`). + /// Bounds the anonymous `acquire` + `cat-file` fan-out across the node (#173, + /// INV-10); a field for the same test-seam reason as `ipfs_max_history_walks`. + pub ipfs_max_legacy_probes: u32, + /// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` will buffer and + /// serve (default `api::ipfs::MAX_SERVED_OBJECT_BYTES`). The serve reads via a + /// blocking `git cat-file` and buffers the whole object; without a bound a large + /// public blob could exhaust memory or block a runtime worker (#173, F6, INV-10). + /// A field for the same test-seam reason as the sibling caps. + pub ipfs_max_served_object_bytes: u64, /// Which forwarded header (if any) the edge is trusted to set, for /// resolving the push limiter's client-IP key. See `GITLAWB_TRUSTED_PROXY`. /// Node-wide; also keys the two peer-sync limiters below. @@ -171,13 +194,6 @@ pub struct AppState { /// (`with_default_max_keys`, reject-before-insert) so a source-key farm cannot grow /// it (INV-15). pub git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency, - /// Per-client-IP rate limiter for `GET /ipfs/{cid}`. The route is publicly - /// reachable and each request can drive a full-history git walk, so it carries a - /// per-IP flood brake in addition to the concurrency cap above — a rate limit - /// bounds request *rate*, the semaphore bounds concurrent slow holds (different - /// axes). Keyed on the resolved client IP via `push_limiter_trust`. Layered on the - /// `/ipfs` route via `rate_limit_by_ip`. - pub ipfs_rate_limiter: RateLimiter, /// The `git` executable the served-git withheld-blob walk spawns. Production is /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's /// process-group teardown in handler tests without mutating the process-global @@ -192,6 +208,20 @@ impl AppState { self.shutdown_tx.subscribe() } + /// Sweep expired entries from every per-IP/DID rate limiter. Driven by the + /// periodic cleanup task so a bounded limiter's key map sheds stale entries + /// instead of sitting near its cap until an inline capacity sweep reclaims + /// them. Every limiter on the state is swept here; adding a new limiter means + /// adding it to this list. + pub(crate) async fn sweep_rate_limiters(&self) { + self.rate_limiter.cleanup().await; + self.create_ip_rate_limiter.cleanup().await; + self.push_rate_limiter.cleanup().await; + self.ipfs_rate_limiter.cleanup().await; + self.sync_trigger_rate_limiter.cleanup().await; + self.peer_write_rate_limiter.cleanup().await; + } + /// Trigger graceful shutdown. Idempotent — calling more than once /// has no effect. Returns `true` if this call was the one that /// flipped the signal. diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 92f41a8d..21fa58e4 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -78,6 +78,10 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), @@ -98,7 +102,6 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 16, ), - ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } @@ -2020,13 +2023,19 @@ mod tests { /// Seed a SHA-256 source repo (public/a.txt + secret/b.txt), bare-clone it /// into each `/tmp//.git` path, and return guards + oids. - /// SHA-256 object format is required: `get_by_cid` resolves a CID whose - /// multihash digest IS the git object id, which only matches in sha256 repos. + /// SHA-256 object format matches production (`--object-format=sha256`) so the + /// oids are 64-hex. A real CID digests the raw object CONTENT (not the git + /// oid), so tests build the request CID with `pin_cid_for` — mirroring the pin + /// path — and `get_by_cid` maps it back to the oid via `pinned_cids` (#173). struct CidFixture { _guards: Vec, secret_oid: String, public_oid: String, secret_tree_oid: String, + public_tree_oid: String, + root_tree_oid: String, + commit_oid: String, + tag_oid: String, } impl Drop for CidFixture { fn drop(&mut self) { @@ -2060,6 +2069,8 @@ mod tests { run(&["config", "user.name", "t"], &src); run(&["add", "."], &src); run(&["commit", "-qm", "seed"], &src); + // Annotated tag of the commit — exercises the "tags stay served" guard. + run(&["tag", "-a", "-m", "annotated", "v1", "HEAD"], &src); let oid = |rev: &str| { let out = Command::new("git") .args(["rev-parse", rev]) @@ -2072,6 +2083,10 @@ mod tests { let secret_oid = oid("HEAD:secret/b.txt"); let public_oid = oid("HEAD:public/a.txt"); let secret_tree_oid = oid("HEAD:secret"); + let public_tree_oid = oid("HEAD:public"); + let root_tree_oid = oid("HEAD^{tree}"); + let commit_oid = oid("HEAD"); + let tag_oid = oid("refs/tags/v1"); let mut guards = vec![src.clone()]; for name in bare_names { let bare = std::path::PathBuf::from("/tmp") @@ -2089,6 +2104,12 @@ mod tests { ], &src, ); + // `git clone --bare` does NOT copy the source repo's local identity, so + // fixtures that create objects directly in the bare repo (`commit-tree`, + // `git tag -a`) abort with "identity unknown" on a CI runner that has no + // ambient/global git identity. Set it explicitly so the suite is portable. + run(&["config", "user.email", "t@t"], &bare); + run(&["config", "user.name", "t"], &bare); } // One guard for the whole /tmp/ tree covers every bare clone. guards.push(std::path::PathBuf::from("/tmp").join(slug)); @@ -2097,300 +2118,3160 @@ mod tests { secret_oid, public_oid, secret_tree_oid, + public_tree_oid, + root_tree_oid, + commit_oid, + tag_oid, } } - /// CID whose sha2-256 multihash digest equals the given 64-hex git oid, so - /// `get_by_cid` decodes it back to that oid and `git cat-file`s it. - fn cid_for_oid(oid_hex: &str) -> String { - use gitlawb_core::cid::Cid; - let bytes = hex::decode(oid_hex).expect("hex oid"); - let arr: [u8; 32] = bytes.as_slice().try_into().expect("32-byte sha256 oid"); - Cid::from_sha256_bytes(&arr).to_string() + /// Record a pin exactly as the production pin path does — read the object's + /// raw bytes (`git cat-file `, no framing), CID them with + /// `Cid::from_git_object_bytes`, and store the `(oid, cid)` row — then return + /// the CID string the node advertises (`gl ipfs list`) and a client sends to + /// `GET /ipfs/{cid}`. Building the CID from the oid instead (the old + /// `cid_for_oid`) produced an identifier that never occurs in production and + /// made the gate assertions vacuous: a real pin CID digests the raw content, + /// not the git oid, so `get_by_cid` resolves it through `pinned_cids` (#173). + async fn pin_cid_for(bare_repo: &std::path::Path, oid: &str, db: &crate::db::Db) -> String { + let (_ty, raw) = crate::git::store::read_object(bare_repo, oid) + .expect("read object bytes") + .expect("object exists in repo"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + // Legacy-style pin (no provenance) so existing CID tests exercise the + // resolver's scan fallback; provenance-path tests pin via `pin_cid_for_repo`. + db.record_pinned_cid(oid, &cid, None) + .await + .expect("record pinned cid"); + cid } - fn cid_router(state: &AppState) -> Router { - Router::new() - .route( - "/ipfs/{cid}", - axum::routing::get(crate::api::ipfs::get_by_cid), - ) - .layer(axum::middleware::from_fn(crate::auth::optional_signature)) - .with_state(state.clone()) + /// Like [`pin_cid_for`] but records the pin's provenance (`repo_id`), so the + /// resolver resolves the CID straight to `repo_id` instead of scanning (#173). + #[allow(dead_code)] // used by the provenance-path resolver tests (P-U3) + async fn pin_cid_for_repo( + bare_repo: &std::path::Path, + oid: &str, + db: &crate::db::Db, + repo_id: &str, + ) -> String { + let (_ty, raw) = crate::git::store::read_object(bare_repo, oid) + .expect("read object bytes") + .expect("object exists in repo"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + db.record_pinned_cid(oid, &cid, Some(repo_id)) + .await + .expect("record pinned cid with provenance"); + cid } - async fn cid_parts(resp: axum::response::Response) -> (StatusCode, String) { - let st = resp.status(); - let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + + /// INV-7 upgrade path for the pin-provenance column (#173, jatmn round 2): a node + /// already past v11 gets `pinned_cids.repo_id` from the NEW v12 migration, and a + /// legacy pin recorded before the column existed survives with NULL provenance (so + /// it falls back to the repo scan). Simulate the pre-v12 node by dropping the + /// column and un-applying v12, seed a legacy row, then re-migrate. RED before the + /// v12 migration exists (the column is never re-added → the SELECT errors); GREEN + /// after. + #[sqlx::test] + async fn pinned_cids_repo_provenance_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v12 shape: drop the provenance column and forget v12 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS repo_id") + .execute(&pool) .await .unwrap(); - (st, String::from_utf8_lossy(&b).to_string()) - } - fn cid_anon(cid: &str) -> Request { - Request::builder() - .method(Method::GET) - .uri(format!("/ipfs/{cid}")) - .body(Body::empty()) - .unwrap() - } - fn cid_signed(kp: &gitlawb_core::identity::Keypair, cid: &str) -> Request { - let path = format!("/ipfs/{cid}"); - let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); - Request::builder() - .method(Method::GET) - .uri(&path) - .header("content-digest", s.content_digest) - .header("signature-input", s.signature_input) - .header("signature", s.signature) - .body(Body::empty()) - .unwrap() + sqlx::query("DELETE FROM schema_migrations WHERE version = 12") + .execute(&pool) + .await + .unwrap(); + + // A legacy pin recorded before provenance existed. + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind("legacyoid") + .bind("legacycid") + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + // Upgrade: re-run migrations → v12 re-adds the column. + state.db.run_migrations().await.expect("migrate to v12"); + + // The legacy pin survives with NULL provenance. + let legacy: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = 'legacyoid'") + .fetch_one(&pool) + .await + .expect("legacy pin row survives the upgrade"); + assert!( + legacy.is_none(), + "a pin recorded before v12 must keep NULL provenance (it falls back to the scan)" + ); + + // A new pin can carry provenance. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind("newoid") + .bind("newcid") + .bind("2026-01-01T00:00:00Z") + .bind("repo-abc") + .execute(&pool) + .await + .unwrap(); + let prov: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = 'newoid'") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + prov.as_deref(), + Some("repo-abc"), + "a pin recorded after v12 carries its source repo_id" + ); } - /// #110: `GET /ipfs/{cid}` must gate a withheld blob by per-caller visibility. - /// RED before U2 (the current handler serves the secret to anon). + /// #173: a pin records the repository it came from; `provenance_for_oid` reads it + /// back; a legacy pin (no repo) reads back None; and first-pinner-owns holds — a + /// second push of the same oid does NOT rewrite provenance (ON CONFLICT DO + /// NOTHING). This is what lets the resolver gate a CID against its ONE source repo. #[sqlx::test] - async fn ipfs_cid_gate_withholds_blob_from_unauthorized(pool: PgPool) { - use crate::db::VisibilityMode; - use gitlawb_core::identity::Keypair; - - let owner = Keypair::generate(); - let owner_did = owner.did().to_string(); - let reader = Keypair::generate(); - let reader_did = reader.did().to_string(); - let stranger = Keypair::generate(); - let slug = owner_did.replace([':', '/'], "_"); - let short = owner_did.split(':').next_back().unwrap().to_string(); + async fn record_pinned_cid_stores_and_reads_provenance(pool: PgPool) { let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["withhold"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - let tree_cid = cid_for_oid(&fx.secret_tree_oid); - let public_cid = cid_for_oid(&fx.public_oid); - state .db - .create_repo(&seed_repo(&owner_did, "withhold")) + .record_pinned_cid("oidA", "cidA", Some("repo-xyz")) .await - .expect("seed repo"); - let rec = state + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("oidA") + .await + .unwrap() + .as_deref(), + Some("repo-xyz"), + "a provenanced pin reads back its source repo_id" + ); + + state .db - .get_repo(&owner_did, "withhold") + .record_pinned_cid("oidB", "cidB", None) .await - .unwrap() .unwrap(); + assert_eq!( + state.db.provenance_for_oid("oidB").await.unwrap(), + None, + "a legacy pin (no repo) has NULL provenance" + ); + + // First-pinner-owns: a later push of the same oid must not rewrite provenance. state .db - .set_visibility_rule( - &rec.id, - "/secret/**", - VisibilityMode::B, - std::slice::from_ref(&reader_did), - &owner_did, - ) + .record_pinned_cid("oidA", "cidA", Some("repo-OTHER")) .await - .expect("deny rule"); - - // anon → withheld blob: must 404, must not leak content. (RED on current handler.) - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&secret_cid)) - .await - .unwrap(), - ) - .await; + .unwrap(); assert_eq!( - st, - StatusCode::NOT_FOUND, - "anon must not read the withheld blob" - ); - assert!( - !body.contains("TOP SECRET"), - "404 body must not leak the secret" + state + .db + .provenance_for_oid("oidA") + .await + .unwrap() + .as_deref(), + Some("repo-xyz"), + "ON CONFLICT DO NOTHING keeps the first repo's provenance" ); - // signed non-reader → 404. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&stranger, &secret_cid)) - .await - .unwrap(), - ) - .await; + // An unpinned oid has no provenance. assert_eq!( - st, - StatusCode::NOT_FOUND, - "non-reader must not read the withheld blob" + state.db.provenance_for_oid("never-pinned").await.unwrap(), + None ); - assert!(!body.contains("TOP SECRET")); - - // owner (signed) → 200 + secret bytes. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&owner, &secret_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "owner reads the withheld blob"); - assert!(body.contains("TOP SECRET"), "owner gets the content"); + } - // listed reader (signed) → 200. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&reader, &secret_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "listed reader reads the blob"); - assert!(body.contains("TOP SECRET")); + /// #173 (provenance, happy path): a CID pinned with provenance resolves straight + /// to its ONE source repo and serves an authorized reader — no repo scan. + #[sqlx::test] + async fn ipfs_cid_provenance_serves_from_pinning_repo(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; - // KTD3: anon tree CID under /secret → 200 (trees/commits are not withheld). - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&tree_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "tree object is served to anon (KTD3)"); + let _fx = seed_cid_repos(&slug, &short, &["provserve"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provserve.git"); + let fx = &_fx; - // R3: public blob anon → 200 (non-withheld content not affected). - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&public_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "public blob stays served"); + // Build the repo FIRST so the pin can carry its id as provenance. + let repo = seed_repo(&owner_did, "provserve"); // public + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; - // R5: a genuine unknown CID also 404, uniform with the withheld 404. - let absent_cid = cid_for_oid(&"ab".repeat(32)); - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&absent_cid)) - .await - .unwrap(), - ) - .await; + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; assert_eq!( st, - StatusCode::NOT_FOUND, - "absent CID 404 (uniform with withheld)" + StatusCode::OK, + "a provenanced public CID serves its content" + ); + assert!( + body.contains("public bytes"), + "the pinning repo's object is served" ); - - // malformed CID → 400 (unchanged). - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon("not-a-cid")) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::BAD_REQUEST, "malformed CID still 400"); } - /// R4: the same object withheld in one repo but public in another is still - /// served from the public copy; the withholding repo is iterated first. + /// #173 (provenance, THE load-bearing one — #124 flip + bounded fan-out): a CID + /// pinned from a PRIVATE repo must gate against that pinning repo (404), NOT serve + /// from a byte-identical PUBLIC copy in another repo. Provenance is strictly more + /// restrictive than the old scan (which served the public copy). RED before the + /// rework (the scan serves the public copy → 200 + leaks the secret bytes); GREEN + /// after (provenance → the private repo → 404, no leak). #[sqlx::test] - async fn ipfs_cid_served_from_public_copy_when_withheld_elsewhere(pool: PgPool) { - use crate::db::VisibilityMode; - use chrono::Utc; + async fn ipfs_cid_provenance_private_denies_despite_public_copy(pool: PgPool) { use gitlawb_core::identity::Keypair; - let owner = Keypair::generate(); let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["withhold", "pubcopy"]); - let secret_cid = cid_for_oid(&fx.secret_oid); + let fx = seed_cid_repos(&slug, &short, &["privsrc", "pubcopy"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privsrc.git"); - // Withholding repo, iterated FIRST (later updated_at; list_all_repos is DESC). - let mut withhold = seed_repo(&owner_did, "withhold"); - withhold.updated_at = Utc::now(); + // Private source repo, built first so the pin carries its id as provenance. + let mut priv_repo = seed_repo(&owner_did, "privsrc"); + priv_repo.is_public = false; state .db - .create_repo(&withhold) + .create_repo(&priv_repo) .await - .expect("withhold repo"); + .expect("seed private repo"); + let cid = pin_cid_for_repo(&priv_bare, &fx.secret_oid, &state.db, &priv_repo.id).await; + + // A PUBLIC repo holds the SAME object (the old scan would serve it). + let pub_repo = seed_repo(&owner_did, "pubcopy"); // public, no rule state .db - .set_visibility_rule( - &withhold.id, - "/secret/**", - VisibilityMode::B, - &[], - &owner_did, - ) + .create_repo(&pub_repo) .await - .expect("deny rule"); - - // Public copy, no rules, iterated AFTER. - let mut pubcopy = seed_repo(&owner_did, "pubcopy"); - pubcopy.updated_at = Utc::now() - chrono::Duration::seconds(60); - state.db.create_repo(&pubcopy).await.expect("pubcopy repo"); + .expect("seed public copy"); - // anon: denied at the withholding repo (continue), served from the public copy. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&secret_cid)) - .await - .unwrap(), - ) - .await; + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; assert_eq!( st, - StatusCode::OK, - "served from the public copy despite the other deny" + StatusCode::NOT_FOUND, + "a provenanced private CID must 404, not serve from a public copy elsewhere (#124 flip)" ); assert!( - body.contains("TOP SECRET"), - "the public copy serves the content" + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld object" ); } - /// Repo-level "/" gate (KTD2a, first continue branch): a fully private repo - /// (is_public=false, no rules) denies anon before any per-blob check; the - /// owner still reads. The path-scoped tests pass the "/" gate and deny at the - /// per-blob stage, so this exercises the coarser repo-level deny separately. + /// #173 (jatmn round 8, F1 — load-bearing): a shared object first pinned from a + /// PRIVATE repo, then pushed again from a PUBLIC repo through the real pin path, + /// must serve by CID to an anonymous caller from the public source. First-pinner- + /// only provenance 404s it (only the private source is known); recording EVERY + /// pin-path source fixes it. The second push hits the already-pinned skip branch, + /// so this proves the skip-branch source insert fires (and does NOT re-pin: /add + /// expect(0)). RED before U1 (anon 404); GREEN after. #[sqlx::test] - async fn ipfs_cid_private_repo_denies_anon_at_repo_gate(pool: PgPool) { + async fn ipfs_cid_multi_source_serves_from_later_public_pinner(pool: PgPool) { use gitlawb_core::identity::Keypair; - let owner = Keypair::generate(); let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["priv"]); - let blob_cid = cid_for_oid(&fx.public_oid); - - let mut rec = seed_repo(&owner_did, "priv"); - rec.is_public = false; - state.db.create_repo(&rec).await.expect("private repo"); + let fx = seed_cid_repos(&slug, &short, &["privfirst", "pubsecond"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privfirst.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubsecond.git"); + + // Private repo pins the object FIRST — it owns the first-pinner provenance. + let mut priv_repo = seed_repo(&owner_did, "privfirst"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private first-pinner"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // A PUBLIC repo pushes the SAME object through the real pin path. The object is + // already pinned, so this hits the already-pinned skip branch, which must record + // the public repo as an additional source without re-pinning (/add expect 0). + let pub_repo = seed_repo(&owner_did, "pubsecond"); // public, no rule + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public second-pinner"); + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &pub_bare, + vec![fx.public_oid.clone()], + &state.db, + &pub_repo.id, + ) + .await; + m.assert_async().await; // asserts /add was NOT called (already pinned) + + // Anonymous CID fetch: the private first source denies, the public second + // source serves → 200. Before F1 only the private source is known → 404. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a shared object must serve by CID from a later public pin-path source (F1)" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + } + + /// #173 (jatmn round 8, F1 — bound, R2): the per-object source set is capped at + /// `MAX_PIN_SOURCES` so an adversary pushing one object from many repos cannot make + /// resolution O(repos). Recording the same oid from `MAX_PIN_SOURCES + 3` distinct + /// repos leaves exactly `MAX_PIN_SOURCES` rows. + #[sqlx::test] + async fn ipfs_cid_pin_sources_capped_at_max(pool: PgPool) { + let state = test_state(pool).await; + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..(cap + 3) { + state + .db + .record_pin_source("capoid", &format!("repo-{i}")) + .await + .expect("record source"); + } + let sources = state.db.pin_sources_for_oid("capoid").await.unwrap(); + assert_eq!( + sources.len() as i64, + cap, + "the per-object source set is capped at MAX_PIN_SOURCES" + ); + } + + /// #173 (jatmn round 8, F1 — availability, grok-4.5 adversarial catch): the resolver's + /// per-object source cap must NEVER evict the first-pinner. A legacy public pin keeps + /// its source in `pinned_cids.repo_id` but not in `pin_repo_sources` (pre-v13 pins, or + /// a pin whose best-effort `record_pin_source` missed). If the cap `LIMIT` were applied + /// to the whole union with a lexicographic order, an attacker could push the same + /// object from `MAX_PIN_SOURCES` repos whose grindable ids sort before the public + /// source and evict it from the window — turning a public CID that served 200 into a + /// 404. This drives exactly that: a legacy public first-pinner plus `MAX_PIN_SOURCES` + /// lower-sorting attacker sources must STILL serve the public object. RED with a + /// whole-union LIMIT (the first-pinner is dropped → 404); GREEN once the first-pinner + /// is always included and the LIMIT caps only the additional sources. + #[sqlx::test] + async fn ipfs_cid_first_pinner_never_evicted_by_lower_sorting_sources(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["pubfirst"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubfirst.git"); + // Public repo whose id sorts AFTER every attacker id below. Legacy shape: the + // source lives in pinned_cids.repo_id only (pin_cid_for_repo records no + // pin_repo_sources row), exactly like a pin from before v13. + let mut pub_repo = seed_repo(&owner_did, "pubfirst"); // public, no rule + pub_repo.id = "zzzzzzzz-pubfirst".to_string(); + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public first-pinner"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &pub_repo.id).await; + + // Attacker fills the whole MAX_PIN_SOURCES window with lower-sorting source ids + // (non-existent repos — their mere presence would evict the first-pinner under a + // whole-union LIMIT). + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..cap { + state + .db + .record_pin_source(&fx.public_oid, &format!("00-attacker-{i:02}")) + .await + .expect("attacker source"); + } + + // The public first-pinner must still serve — never evicted by the cap window. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "the first-pinner public source must never be evicted by lower-sorting attacker sources (F1 availability)" + ); + assert!( + body.contains("public bytes"), + "the public object is served from the first-pinner" + ); + } + + /// INV-7 upgrade path for the F1 `pin_repo_sources` table (#173, jatmn round 8): a + /// node already past v12 gets the table from the NEW v13 migration. Simulate the + /// pre-v13 node by dropping the table and un-applying v13, then re-migrate and + /// assert a source row round-trips. RED before the v13 migration exists. + #[sqlx::test] + async fn pin_repo_sources_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + sqlx::query("DROP TABLE IF EXISTS pin_repo_sources") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 13") + .execute(&pool) + .await + .unwrap(); + state.db.run_migrations().await.expect("re-migrate"); + state + .db + .record_pin_source("upgradeoid", "repo-upg") + .await + .expect("record after re-migrate"); + assert_eq!( + state.db.pin_sources_for_oid("upgradeoid").await.unwrap(), + vec!["repo-upg".to_string()], + "the v13 pin_repo_sources table is present after upgrade" + ); + } + + /// #173 (jatmn round 8, F2 — load-bearing): a legacy `pinned_cids` row keyed on a + /// PROVIDER CID (Pinata/Kubo dag-pb — every release before this branch stored the + /// provider CID as the resolver key, not the raw-content CID) must NOT serve raw git + /// bytes that do not hash to the requested CID. `get_by_cid` recomputes the CID over + /// the served bytes and refuses to serve on mismatch. Seeded with a RAW SQL INSERT + /// because the current helpers store the raw CID, so a helper-seeded row is already + /// correct-shape and the RED assertion would be vacuous (INV-21). RED before U2 + /// (serves the git bytes → 200); GREEN after (not served, no bytes egress). + #[sqlx::test] + async fn ipfs_cid_legacy_provider_cid_row_not_served(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["provsrc"]); + let repo = seed_repo(&owner_did, "provsrc"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + + // A valid sha2-256 CID whose digest is NOT the object's raw-content digest — + // stands in for a Pinata/Kubo dag-pb provider CID (the legacy resolver key). + let provider_cid = gitlawb_core::cid::Cid::from_git_object_bytes( + b"a decoy object whose CID is not the served object's CID", + ) + .to_string(); + + // Legacy-shape row: cid = the PROVIDER CID (raw SQL — the helpers now store the + // raw CID and cannot reproduce this shape). The object itself is public+servable. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&fx.public_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + // Requesting the provider CID resolves the row and passes the repo gate, but the + // served bytes hash to a DIFFERENT CID, so the integrity check must withhold them. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&provider_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st, + StatusCode::OK, + "a provider-CID legacy row must not serve raw git bytes (F2)" + ); + assert!( + !body.contains("public bytes"), + "the mismatched bytes must not egress" + ); + } + + /// #173 (jatmn round 8, F6 — INV-10 cost guard): the serve path buffers the object via + /// a blocking `cat-file`; an object larger than `ipfs_max_served_object_bytes` must be + /// WITHHELD (rejected by the size precheck, never buffered), with zero body bytes + /// egressed. Under the cap it serves unchanged. The oversize-reject counter guards it + /// both ways: a removed size precheck serves the object and leaves the counter at 0. + #[sqlx::test] + async fn ipfs_cid_f6_oversized_object_withheld(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["big"]); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("big.git"); + let repo = seed_repo(&owner_did, "big"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Cap below the object size ("public bytes\n" = 13 bytes) → withheld. + state.ipfs_max_served_object_bytes = 5; + crate::api::ipfs::reset_oversize_rejects(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_ne!( + st, + StatusCode::OK, + "an object over the size cap must not serve (F6)" + ); + assert!( + !body.contains("public bytes"), + "no object bytes egress for an over-cap object" + ); + assert_eq!( + crate::api::ipfs::oversize_rejects(), + 1, + "the oversized object was rejected by the size precheck" + ); + + // Control: raise the cap above the object size → serves unchanged. + state.ipfs_max_served_object_bytes = crate::api::ipfs::MAX_SERVED_OBJECT_BYTES; + crate::api::ipfs::reset_oversize_rejects(); + let (st2, body2) = + cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st2, + StatusCode::OK, + "under the cap the object serves normally" + ); + assert!( + body2.contains("public bytes"), + "the served body is the object's bytes" + ); + assert_eq!( + crate::api::ipfs::oversize_rejects(), + 0, + "no oversize reject under the cap" + ); + } + + /// #173 (provenance, INV-11): a quarantined pinning repo must 404 by CID even for + /// its own owner — quarantine hard-drops before the visibility gate on the + /// provenance path too. The owner-signed 404 is the load-bearing negative (a + /// visibility-only gate would Allow the owner). + #[sqlx::test] + async fn ipfs_cid_provenance_quarantined_repo_404_even_owner(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["quarsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("quarsrc.git"); + let repo = seed_repo(&owner_did, "quarsrc"); // public + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Baseline: before quarantine the provenanced CID serves (proves the path works). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "provenanced CID serves before quarantine" + ); + + state + .db + .set_repo_quarantine(&repo.id, true) + .await + .expect("quarantine"); + + for req in [cid_anon(&cid), cid_signed(&owner, &cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a quarantined pinning repo must 404 by CID (anon + owner)" + ); + assert!( + !body.contains("public bytes"), + "the 404 body must not leak quarantined content" + ); + } + } + + /// #173 (provenance, bounded — must NOT fall back to the scan): a CID whose + /// provenance points at a repo that no longer exists must 404 rather than scan + /// every repo and serve a byte-identical public copy. Falling back to the scan + /// would reopen the O(repos) anonymous fan-out the provenance rework closes. RED + /// before the rework (the scan serves the public copy → 200); GREEN after. + #[sqlx::test] + async fn ipfs_cid_provenance_missing_repo_404_no_scan_fallback(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["gonesrc", "pubcopy2"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("gonesrc.git"); + + // Pin with provenance = a repo_id that is never created (deleted/absent). + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, "nonexistent-repo-id").await; + + // A public repo holds the SAME object (the old scan would serve it). + let pub_repo = seed_repo(&owner_did, "pubcopy2"); + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public copy"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a provenance pointing at a missing repo must 404, not fall back to the scan" + ); + } + + /// #173 (provenance, path-scoped WALK gate): the #135/#173 per-object gates must + /// run on the NEW provenance path, not only the legacy scan. A provenanced pin from + /// a repo under a `/secret/**` rule runs `allowed_blob_set_for_caller` via the shared + /// gate: a withheld secret blob 404s to anon (no byte leak); the allowed reader gets + /// it. Exercises the walk gate on the provenance path in BOTH directions. + #[sqlx::test] + async fn ipfs_cid_provenance_path_scoped_walk_gates_withheld_blob(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["provwalk"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provwalk.git"); + let repo = seed_repo(&owner_did, "provwalk"); // public at "/" + state.db.create_repo(&repo).await.expect("seed repo"); + // /secret/** Mode B with the reader allowed → the secret blob walk gates by caller. + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + + // Anon: the walk denies the secret blob → 404, no leak. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a withheld secret blob 404s to anon on the provenance path (walk gate runs)" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld blob" + ); + + // Allowed reader: the walk includes the secret blob → 200 with content. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&reader, &cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "an allowed reader gets the secret blob via the provenance walk gate" + ); + assert!( + body.contains("TOP SECRET"), + "the allowed reader receives the content" + ); + } + + /// #173: the pinata pin path stores the locally-computed raw CID in the + /// resolver-key `cid` column and the provider CID in `pinata_cid`, and its ON + /// CONFLICT COALESCE fills a NULL provenance without overwriting an existing one + /// (first-pinner-owns). On conflict `cid` is left untouched so a prior local pin's + /// raw CID is never clobbered by a provider CID. + #[sqlx::test] + async fn record_pinata_cid_stores_and_coalesces_provenance(pool: PgPool) { + let state = test_state(pool).await; + + // A new row created via the pinata path carries provenance, and stores the + // raw CID in `cid` with the provider CID in `pinata_cid`. + state + .db + .record_pinata_cid("po1", "rawcid1", "pcid1", Some("repoA")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po1").await.unwrap().as_deref(), + Some("repoA") + ); + let po1 = state + .db + .list_pinned_cids() + .await + .unwrap() + .into_iter() + .find(|r| r.sha256_hex == "po1") + .expect("po1 row exists"); + assert_eq!(po1.cid, "rawcid1", "resolver-key cid is the raw CID"); + assert_eq!( + po1.pinata_cid.as_deref(), + Some("pcid1"), + "the provider CID is kept in pinata_cid" + ); + + // An existing NULL-provenance row: the pinata COALESCE fills it, and the + // prior local pin's `cid` is left untouched (not overwritten by the raw arg). + state + .db + .record_pinned_cid("po2", "localcid2", None) + .await + .unwrap(); + state + .db + .record_pinata_cid("po2", "rawcid2", "pcid2", Some("repoB")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po2").await.unwrap().as_deref(), + Some("repoB"), + "pinata fills a NULL provenance" + ); + let po2 = state + .db + .list_pinned_cids() + .await + .unwrap() + .into_iter() + .find(|r| r.sha256_hex == "po2") + .expect("po2 row exists"); + assert_eq!( + po2.cid, "localcid2", + "on conflict the prior local pin's cid is left untouched" + ); + + // An existing provenance: the pinata COALESCE must NOT overwrite it. + state + .db + .record_pinned_cid("po3", "cid3", Some("repoX")) + .await + .unwrap(); + state + .db + .record_pinata_cid("po3", "rawcid3", "pcid3", Some("repoY")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po3").await.unwrap().as_deref(), + Some("repoX"), + "pinata COALESCE keeps the first-pinner's provenance" + ); + } + + /// #173 (jatmn, F4, load-bearing security): a Pinata-first pin (no prior local pin) + /// must make the resolver key (`pinned_cids.cid`) the locally-computed raw CID, NOT + /// the provider CID. Pinata wraps the bytes in dag-pb/UnixFS, so its returned CID + /// does not hash the raw content; if it became the resolver key, `/ipfs/{provider_cid}` + /// would serve raw git bytes that do not hash to it, breaking raw content-addressing. + /// Assert `oids_for_cid(raw_cid)` finds the sha AND `oids_for_cid(provider_cid)` does NOT. + #[sqlx::test] + async fn record_pinata_cid_resolver_key_is_raw_not_provider(pool: PgPool) { + let state = test_state(pool).await; + + let bytes = b"raw git object content for pinata-first pin"; + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(bytes).to_string(); + // A distinct provider CID (a dag-pb wrapper CID Pinata would return). + let provider_cid = "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"; + assert_ne!( + raw_cid, provider_cid, + "the provider CID must differ from the raw CID for this test to be meaningful" + ); + + // Pinata-first: no prior local pin, so this INSERT creates the row. + state + .db + .record_pinata_cid("pfsha", &raw_cid, provider_cid, Some("repoP")) + .await + .unwrap(); + + // The raw CID resolves to the sha. + assert_eq!( + state.db.oids_for_cid(&raw_cid).await.unwrap(), + vec!["pfsha".to_string()], + "the locally-computed raw CID is the resolver key" + ); + // The provider (dag-pb) CID must NOT resolve raw bytes. + assert!( + state + .db + .oids_for_cid(provider_cid) + .await + .unwrap() + .is_empty(), + "the provider dag-pb CID must never resolve raw git bytes" + ); + } + + /// #173 (end-to-end pin wiring): `pin_new_objects` records the repo_id it is given + /// as the pin's provenance. Drives the real pin path against a mocked IPFS `/add` + /// endpoint (so `pin_git_object` succeeds) and asserts `provenance_for_oid` returns + /// the repo — closing the gap between the push handler's threading and the DB write. + #[sqlx::test] + async fn pin_new_objects_records_provenance(pool: PgPool) { + let state = test_state(pool).await; + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovtest"}"#) + .expect_at_least(1) + .create_async() + .await; + + let fx = seed_cid_repos("provpin_e2e", "ppe2e", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("provpin_e2e") + .join("pinsrc.git"); + + let pinned = crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + vec![fx.public_oid.clone()], + &state.db, + "repoZ", + ) + .await; + assert!( + !pinned.is_empty(), + "the object was pinned via the real pin path" + ); + m.assert_async().await; + assert_eq!( + state + .db + .provenance_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some("repoZ"), + "pin_new_objects records the repo_id it was given as the pin's provenance" + ); + } + + /// #173 (jatmn, F2): a legacy pin with NULL provenance backfills its source + /// via `backfill_pin_provenance`, and the `AND repo_id IS NULL` guard preserves + /// first-pinner-owns (a non-NULL provenance is left untouched). + #[sqlx::test] + async fn backfill_pin_provenance_fills_null_keeps_existing(pool: PgPool) { + let state = test_state(pool).await; + + // A legacy pin: no provenance recorded. + state + .db + .record_pinned_cid("legacy_oid", "legacy_cid", None) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("legacy_oid").await.unwrap(), + None, + "a legacy pin starts with NULL provenance" + ); + + // Backfill sets the NULL provenance. + state + .db + .backfill_pin_provenance("legacy_oid", "repo-src") + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("legacy_oid") + .await + .unwrap() + .as_deref(), + Some("repo-src"), + "backfill fills a NULL provenance from the known source" + ); + + // A pin that already has provenance: backfill must NOT overwrite it. + state + .db + .record_pinned_cid("owned_oid", "owned_cid", Some("repo-first")) + .await + .unwrap(); + state + .db + .backfill_pin_provenance("owned_oid", "repo-second") + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("owned_oid") + .await + .unwrap() + .as_deref(), + Some("repo-first"), + "the AND repo_id IS NULL guard keeps the first-pinner's provenance" + ); + } + + /// #173 (jatmn, F2, load-bearing): an object already pinned with NULL provenance + /// (a pre-provenance legacy pin) acquires its source when `pin_new_objects` sees + /// it again. The already-pinned skip path must backfill rather than leave the + /// object stuck on the O(repos) scan fallback — and it must NOT re-pin the bytes + /// (no IPFS `/add` call, the object is already on IPFS). + #[sqlx::test] + async fn pin_new_objects_backfills_legacy_null_provenance(pool: PgPool) { + let state = test_state(pool).await; + + let fx = seed_cid_repos("provpin_backfill", "ppbf", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("provpin_backfill") + .join("pinsrc.git"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes( + &crate::git::store::read_object(&bare, &fx.public_oid) + .expect("read object bytes") + .expect("object exists") + .1, + ) + .to_string(); + + // Legacy pin: the object is already recorded with NULL provenance. + state + .db + .record_pinned_cid(&fx.public_oid, &cid, None) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid(&fx.public_oid).await.unwrap(), + None, + "the object starts as a legacy pin with NULL provenance" + ); + + // Mock IPFS `/add` and require it is NOT called: the already-pinned object + // must be backfilled, never re-pinned. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + + let pinned = crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + vec![fx.public_oid.clone()], + &state.db, + "repoBF", + ) + .await; + + assert!( + pinned.is_empty(), + "an already-pinned object is not re-pinned (no bytes returned)" + ); + m.assert_async().await; // asserts /add was called 0 times + assert_eq!( + state + .db + .provenance_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some("repoBF"), + "pin_new_objects backfills the legacy pin's NULL provenance" + ); + } + + /// #173 (provenance-path throttle): a walk-requiring provenanced candidate whose + /// per-IP walk quota is spent returns 429 (the provenance arm's Throttled outcome, + /// then the fall-through). quota=1, keyed on XFF. The first reader request runs the + /// walk and spends the token; the second from the same IP is throttled → 429. + #[sqlx::test] + async fn ipfs_cid_provenance_walk_throttle_returns_429(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["provthrottle"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provthrottle.git"); + let repo = seed_repo(&owner_did, "provthrottle"); + state.db.create_repo(&repo).await.expect("seed repo"); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + + // 1st reader request runs the walk (reader is allowed) and spends the token. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "1st provenance walk from the IP serves"); + + // 2nd request from the same IP: the walk is throttled → 429 (provenance path). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "a throttled provenance walk returns 429" + ); + } + + /// #173 (multi-oid dispatch, mixed provenance + legacy): one CID mapping to a + /// provenanced-then-denied oid AND a legacy (NULL-provenance) oid must still resolve + /// to the legacy-servable copy — the provenance arm's skip does not abort the loop. + #[sqlx::test] + async fn ipfs_cid_mixed_provenance_and_legacy_serves_legacy(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["mixpriv", "mixpub"]); + + // Private repo holds secret_oid, pinned with provenance = itself (denies anon). + let mut priv_repo = seed_repo(&owner_did, "mixpriv"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + // Public repo holds public_oid, legacy pin (NULL provenance -> scan serves it). + let pub_repo = seed_repo(&owner_did, "mixpub"); + state.db.create_repo(&pub_repo).await.expect("seed public"); + + // One REAL CID (the non-unique cid index) maps to BOTH oids: the public oid as a + // legacy (NULL) pin, and the secret oid provenanced to the private repo. + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("mixpub.git"); + let shared_cid = pin_cid_for(&pub_bare, &fx.public_oid, &state.db).await; + state + .db + .record_pinned_cid(&fx.secret_oid, &shared_cid, Some(&priv_repo.id)) + .await + .unwrap(); + + // Anon: secret_oid (provenance -> private -> denied), public_oid (legacy -> scan + // -> public -> served). Resolves to the public copy regardless of oid order. + let resp = cid_router(&state) + .oneshot(cid_anon(&shared_cid)) + .await + .unwrap(); + let served = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "a CID mixing a provenanced-denied oid and a legacy-servable oid resolves" + ); + assert_eq!( + served.as_deref(), + Some(fx.public_oid.as_str()), + "the served object is the legacy public oid" + ); + assert!( + body.contains("public bytes"), + "the public content is served" + ); + } + + // ---- #173 round 3: legacy (NULL-provenance) scan bound + 503-on-truncation ---- + // The provenance path targets one repo and is already bounded. These cover the + // legacy scan fallback, where an anonymous request could otherwise fan out to + // O(repos) `acquire` + `cat-file` probes (F1) and a walk-cap truncation could + // false-404 an object that may be readable (F2). The bound is a per-request probe + // BUDGET, not a per-IP brake: a walk-free public fetch stays un-rate-limited + // (ipfs_walk_rate_limited_per_source), while the expensive walk keeps its IP brake. + + /// T1 (F1): the probe budget gates BEFORE `acquire`/`cat-file`, so it genuinely + /// bounds the fan-out — a repo past the budget is never probed, even one that + /// WOULD serve. With the budget at 0, a PUBLIC legacy copy that would otherwise + /// serve 200 is not probed at all → 503 truncated (absence unproven). RED before + /// the budget check (the repo is probed and serves 200). + #[sqlx::test] + async fn ipfs_cid_legacy_probe_budget_gates_before_serving(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 0; // probe nothing → any legacy candidate truncates + + let fx = seed_cid_repos(&slug, &short, &["pubprobe"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubprobe.git"); + let repo = seed_repo(&owner_did, "pubprobe"); // public, no path rule → would serve + state.db.create_repo(&repo).await.expect("seed repo"); + // Legacy pin (NULL provenance) → resolver takes the scan fallback. + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "the probe budget gates before the probe: a servable copy past the budget is not reached → 503" + ); + } + + /// T7 (F1/F3 pre-limit): EVERY legacy probe is braked on the source IP from the + /// FIRST one, so a hostile caller cannot repeatedly force the whole-node `acquire` + /// fan-out across requests (each cold `acquire` is a Tigris round-trip, INV-10). + /// Since #173-F3 (jatmn) there is no free budget: a single-repo legacy scan is + /// itself charged. quota=1 keyed on XFF, one PUBLIC legacy copy that serves + /// walk-free (never touches the walk brake), so the second same-IP request can only + /// be shed by the probe brake: req1 serves and spends the token, req2 → 429. RED + /// before the probe brake (req2 serves 200). The cross-request bound this proves is + /// exactly the amplification F3 closes. + #[sqlx::test] + async fn ipfs_cid_legacy_fanout_braked_on_ip_past_free_budget(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["fanout"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("fanout.git"); + let repo = seed_repo(&owner_did, "fanout"); // public, no path rule → walk-free serve + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; // legacy pin + + let (st1, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st1, + StatusCode::OK, + "1st legacy fan-out probe from the IP serves" + ); + + let (st2, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st2, + StatusCode::TOO_MANY_REQUESTS, + "with no free budget, a repeat fan-out from the same IP is braked at the first probe" + ); + } + + /// F3 (jatmn, across-request amplification): the pre-fix free-probe budget was + /// PER REQUEST, so a caller could repeat a known NULL-provenance CID and force a + /// fresh batch of `acquire` + `cat-file` probes every request with zero limiter + /// contact, unbounded anonymous amplification against Tigris. Charging every + /// legacy probe from the first one makes those probes accumulate against the + /// per-IP `ipfs_rate_limiter` ACROSS requests. Four repos, none holding the CID, + /// so a full scan probes all four; the per-IP budget is sized to exactly ONE such + /// scan (4 tokens). req1 (a genuine absence) fully scans and 404s, spending the + /// budget; req2 from the SAME IP is shed at the first probe → 429 (it never + /// re-runs the four `acquire` probes). RED with the old free carve-out restored: + /// req2 re-scans un-braked and 404s again (the amplification stays open). This is + /// the load-bearing across-request bound F3 asks for. + #[sqlx::test] + async fn ipfs_cid_legacy_fanout_bounded_across_requests(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Budget = one full scan of the four seeded repos. A repeat scan from the same + // IP then finds it spent. Keyed on XFF so `oneshot` can choose the source IP. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(4, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let names = ["a0", "a1", "a2", "a3"]; + let _fx = seed_cid_repos(&slug, &short, &names); + for n in names { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo → each probed repo misses, + // so req1 scans all four (spending the four-token budget) and 404s cleanly. + let bogus_oid = "0".repeat(64); + let cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-across-requests").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st1, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st1, + StatusCode::NOT_FOUND, + "1st scan completes under budget: a genuine absence is a definitive 404" + ); + + let (st2, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st2, + StatusCode::TOO_MANY_REQUESTS, + "2nd same-IP scan is shed at the first probe (429), not re-run un-braked: the across-request amplification is closed" + ); + } + + /// #173 (jatmn round 8, F3 — INV-10 cost guard): an already-throttled source's + /// legacy NULL-provenance request must be shed by the non-consuming admission peek + /// BEFORE the O(repos) `scan_ctx` preload runs — not after, where the per-probe + /// brake sits. The preload-query counter proves it both ways: 0 for the throttled + /// replay, 1 for an unthrottled source. RED if the peek is removed (the preload runs + /// while throttled → count 1). The two existing `_fanout_` tests confirm the per- + /// probe consuming charge is untouched (no double-charge, no under-charge). + #[sqlx::test] + async fn ipfs_cid_f3_throttled_source_skips_preload(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Budget 1, keyed on XFF so `oneshot` can choose the source IP. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let _fx = seed_cid_repos(&slug, &short, &["r0"]); + state + .db + .create_repo(&seed_repo(&owner_did, "r0")) + .await + .expect("seed repo"); + // A legacy pin absent from every repo → the scan probes and 404s (spending the + // one token on the first probe). + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"f3-absent").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("legacy pin"); + + // Req1 from 9.9.9.9 spends the one token (and runs the preload once). + let _ = cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(); + + // Measure the throttled replay: the peek must shed it before the preload runs. + crate::api::ipfs::reset_preload_queries(); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "an already-throttled legacy replay is 429" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 0, + "a throttled source must NOT run the O(repos) preload (F3): shed before scan_ctx" + ); + + // Control: an unthrottled source (a different IP) still runs the preload once — + // the peek must not over-block. + crate::api::ipfs::reset_preload_queries(); + let _ = cid_router(&state) + .oneshot(cid_anon_xff(&cid, "8.8.8.8")) + .await + .unwrap(); + assert_eq!( + crate::api::ipfs::preload_queries(), + 1, + "an unthrottled source runs the preload once (the peek must not over-block)" + ); + } + + /// T2 (F1): the legacy scan is bounded per request. With the probe ceiling shrunk + /// to 2 and 3 candidate repos none of which hold the object, the 3rd repo is never + /// probed and the search is reported truncated → 503, not an unbounded fan-out. + /// RED before the probe cap (all 3 probe, none serve, definitive 404). + #[sqlx::test] + async fn ipfs_cid_legacy_scan_probe_cap_truncates_to_503(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 2; + + let _fx = seed_cid_repos(&slug, &short, &["r0", "r1", "r2"]); + for n in ["r0", "r1", "r2"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo: each probed repo misses, + // so the cap (not a hit) decides the outcome. + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-t2").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "a scan truncated by the probe cap is a retryable 503, not a definitive 404" + ); + } + + /// T3 (F2): a walk-cap truncation must not false-404. Walk ceiling shrunk to 1; + /// two public repos each carry a path-scoped rule over the object and deny anon. + /// The 1st spends the single walk (deny), the 2nd is skipped at the cap — the + /// resolver did NOT prove the object unreadable everywhere, so 503, not 404. + /// RED before the walk-cap `truncated` flag (returns the opaque 404). + #[sqlx::test] + async fn ipfs_cid_legacy_walk_cap_truncates_to_503(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_history_walks = 1; + + let fx = seed_cid_repos(&slug, &short, &["wa", "wb"]); + for n in ["wa", "wb"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + } + // Legacy pin of the path-scoped secret blob (present in both repos, denies anon). + let bare_wa = std::path::PathBuf::from("/tmp").join(&slug).join("wa.git"); + let cid = pin_cid_for(&bare_wa, &fx.secret_oid, &state.db).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "the walk cap truncated the scan, so absence is unproven → 503, not a false 404" + ); + } + + /// T4 (must-not over-fire): a legacy CID genuinely absent from every repo on a + /// node UNDER the probe cap still returns the definitive 404 — the 503 fires only + /// on real truncation, never as a blanket replacement for not-found. + #[sqlx::test] + async fn ipfs_cid_legacy_true_absence_stays_404(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 8; // well above the single repo → no truncation + + let _fx = seed_cid_repos(&slug, &short, &["only"]); + let repo = seed_repo(&owner_did, "only"); + state.db.create_repo(&repo).await.expect("seed repo"); + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-t4").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a fully-scanned genuine absence is a definitive 404, not a 503" + ); + } + + /// T5 (provenance path untouched): the probe cap governs ONLY the legacy scan. + /// With the cap set to 0 (which would truncate any legacy probe immediately) a + /// PROVENANCED pin still resolves to its one repo and serves 200 — proving the + /// `legacy_scan=false` guard exempts the provenance path. RED if the guard were + /// dropped (provenance would truncate to 503). + #[sqlx::test] + async fn ipfs_cid_provenance_serves_despite_zero_probe_cap(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 0; // would truncate every LEGACY probe + + let fx = seed_cid_repos(&slug, &short, &["provonly"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provonly.git"); + let repo = seed_repo(&owner_did, "provonly"); // public, no path rule + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "the provenance path ignores the legacy probe cap and serves" + ); + } + + fn cid_router(state: &AppState) -> Router { + Router::new() + .route( + "/ipfs/{cid}", + axum::routing::get(crate::api::ipfs::get_by_cid), + ) + .layer(axum::middleware::from_fn(crate::auth::optional_signature)) + .with_state(state.clone()) + } + async fn cid_parts(resp: axum::response::Response) -> (StatusCode, String) { + let st = resp.status(); + let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + (st, String::from_utf8_lossy(&b).to_string()) + } + /// Raw body bytes (NOT lossy-decoded). A git tree body stores each child oid + /// as 32 RAW bytes that `from_utf8_lossy` mangles to U+FFFD, so a hex + /// `contains` check on `cid_parts`'s String is vacuous. #135 deny tests must + /// witness the leak on these raw bytes. + async fn cid_bytes(resp: axum::response::Response) -> (StatusCode, Vec) { + let st = resp.status(); + let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + (st, b.to_vec()) + } + /// True if `needle` appears as a contiguous byte subsequence of `haystack`. + fn bytes_contain(haystack: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() && haystack.windows(needle.len()).any(|w| w == needle) + } + fn cid_anon(cid: &str) -> Request { + Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap() + } + /// Anonymous CID request carrying `x-forwarded-for: ` — an anon caller with a + /// resolvable source IP, so the per-IP walk brake keys on it (the walk still + /// denies anon at a path rule). + fn cid_anon_xff(cid: &str, xff_ip: &str) -> Request { + Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .header("x-forwarded-for", xff_ip) + .body(Body::empty()) + .unwrap() + } + fn cid_signed(kp: &gitlawb_core::identity::Keypair, cid: &str) -> Request { + let path = format!("/ipfs/{cid}"); + let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); + Request::builder() + .method(Method::GET) + .uri(&path) + .header("content-digest", s.content_digest) + .header("signature-input", s.signature_input) + .header("signature", s.signature) + .body(Body::empty()) + .unwrap() + } + /// Signed CID request carrying `x-forwarded-for: `. Used by the walk + /// rate-limit test to key the per-IP limiter off a chosen source under + /// `TrustedProxy::XForwardedFor` (the request goes through `oneshot`, which + /// leaves no socket peer, so the header is the only key source). + fn cid_signed_xff( + kp: &gitlawb_core::identity::Keypair, + cid: &str, + xff_ip: &str, + ) -> Request { + let path = format!("/ipfs/{cid}"); + let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); + Request::builder() + .method(Method::GET) + .uri(&path) + .header("content-digest", s.content_digest) + .header("signature-input", s.signature_input) + .header("signature", s.signature) + .header("x-forwarded-for", xff_ip) + .body(Body::empty()) + .unwrap() + } + + /// #110: `GET /ipfs/{cid}` must gate a withheld blob by per-caller visibility. + /// RED before U2 (the current handler serves the secret to anon). + #[sqlx::test] + async fn ipfs_cid_gate_withholds_blob_from_unauthorized(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let stranger = Keypair::generate(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + // Request CIDs are the production pin CIDs (content-hash), recorded in + // pinned_cids so get_by_cid resolves each back to its oid (#173). + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + let tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + let root_tree_cid = pin_cid_for(&bare, &fx.root_tree_oid, &state.db).await; + let public_tree_cid = pin_cid_for(&bare, &fx.public_tree_oid, &state.db).await; + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + let tag_cid = pin_cid_for(&bare, &fx.tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "withhold")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "withhold") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("deny rule"); + + // anon → withheld blob: must 404, must not leak content. (RED on current handler.) + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "anon must not read the withheld blob" + ); + assert!( + !body.contains("TOP SECRET"), + "404 body must not leak the secret" + ); + + // signed non-reader → 404. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&stranger, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "non-reader must not read the withheld blob" + ); + assert!(!body.contains("TOP SECRET")); + + // owner (signed) → 200 + secret bytes. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "owner reads the withheld blob"); + assert!(body.contains("TOP SECRET"), "owner gets the content"); + + // listed reader (signed) → 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&reader, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "listed reader reads the blob"); + assert!(body.contains("TOP SECRET")); + + // #135: anon tree CID under withheld /secret → 404. The 404 body is an opaque + // error string (never the object), so status is the load-bearing deny check; + // the real leak witness is the CONTRAST with the reader below, who DOES get a + // 200 carrying the child structure that anon is denied. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "withheld subtree tree must not be served to anon (#135)" + ); + + // Over-denial guard + positive leak witness: the listed reader (signed) DOES + // read the withheld subtree's tree, and its body carries the exact child + // structure anon was denied — the child filename plus the child oid as the 32 + // RAW bytes a git tree stores (witnessed on raw bytes, since cid_parts's lossy + // decode would mangle them). This proves b.txt / secret_raw are the real leak + // markers and that the anon 404 above actually withheld them. + let secret_raw = hex::decode(&fx.secret_oid).expect("hex oid"); + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_signed(&reader, &tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "listed reader reads the withheld subtree tree" + ); + assert!( + bytes_contain(&body, b"b.txt") && bytes_contain(&body, &secret_raw), + "reader's tree body carries the child filename and raw child oid" + ); + + // Root tree (path "/") stays served to anon who passes the "/" gate. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&root_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "root tree stays served (must-serve)"); + + // /public subtree tree stays served to anon (allowed path). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "public subtree tree stays served"); + + // Commit and annotated tag objects stay served (unchanged by #135). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "commit object stays served"); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&tag_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "tag object stays served"); + + // R3: public blob anon → 200 (non-withheld content not affected). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "public blob stays served"); + + // R5: a genuine unknown CID also 404, uniform with the withheld 404. A + // well-formed pin-style CID that was never recorded in pinned_cids, so the + // oid_for_cid resolve misses (the production not-found path). + let absent_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"never pinned to this node").to_string(); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&absent_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "absent CID 404 (uniform with withheld)" + ); + + // malformed CID → 400 (unchanged). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon("not-a-cid")) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "malformed CID still 400"); + } + + /// R4: the same object withheld in one repo but public in another is still + /// served from the public copy; the withholding repo is iterated first. + #[sqlx::test] + async fn ipfs_cid_served_from_public_copy_when_withheld_elsewhere(pool: PgPool) { + use crate::db::VisibilityMode; + use chrono::Utc; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold", "pubcopy"]); + // Same content in both clones -> same oid/CID; read from either. + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + + // Withholding repo, iterated FIRST (later updated_at; list_all_repos is DESC). + let mut withhold = seed_repo(&owner_did, "withhold"); + withhold.updated_at = Utc::now(); + state + .db + .create_repo(&withhold) + .await + .expect("withhold repo"); + state + .db + .set_visibility_rule( + &withhold.id, + "/secret/**", + VisibilityMode::B, + &[], + &owner_did, + ) + .await + .expect("deny rule"); + + // Public copy, no rules, iterated AFTER. + let mut pubcopy = seed_repo(&owner_did, "pubcopy"); + pubcopy.updated_at = Utc::now() - chrono::Duration::seconds(60); + state.db.create_repo(&pubcopy).await.expect("pubcopy repo"); + + // anon: denied at the withholding repo (continue), served from the public copy. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "served from the public copy despite the other deny" + ); + assert!( + body.contains("TOP SECRET"), + "the public copy serves the content" + ); + } + + /// Repo-level "/" gate (KTD2a, first continue branch): a fully private repo + /// (is_public=false, no rules) denies anon before any per-blob check; the + /// owner still reads. The path-scoped tests pass the "/" gate and deny at the + /// per-blob stage, so this exercises the coarser repo-level deny separately. + #[sqlx::test] + async fn ipfs_cid_private_repo_denies_anon_at_repo_gate(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["priv"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("priv.git"); + let blob_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + let mut rec = seed_repo(&owner_did, "priv"); + rec.is_public = false; + state.db.create_repo(&rec).await.expect("private repo"); + + // anon → repo-level deny → 404, no content leaked. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&blob_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "anon denied at a private repo's / gate" + ); + assert!(!body.contains("public bytes"), "404 must not leak content"); + + // owner-signed → 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &blob_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "owner reads their private repo's object" + ); + assert!(body.contains("public bytes"), "owner gets the content"); + } + + /// Fail-closed walk-error arm: if `withheld_blob_oids` errors (here, a ref + /// pointing at a non-tree-ish blob, which `git ls-tree -r` cannot traverse — + /// the same induction as `visibility_pack::fails_closed_when_a_ref_cannot_be_traversed`), + /// the handler skips the whole repo rather than serving. Asserts no leak of the + /// withheld blob AND that even the *public* blob in that repo is withheld — the + /// latter distinguishes fail-closed-skip from normal per-blob withholding and + /// would serve 200 if the error arm wrongly proceeded. + #[sqlx::test] + async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + // Recorded pins so get_by_cid resolves each CID to its oid and reaches the + // walk; the 404s below are then the fail-closed skip, not a table miss. + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + // Force the withheld walk to fail closed: a ref pointing at a blob (not + // tree-ish) makes `git ls-tree -r` error, which `withheld_blob_oids` + // propagates as Err → the handler's `Ok(Err)` arm skips the repo. + std::fs::write( + bare.join("refs/heads/blobref"), + format!("{}\n", fx.secret_oid), + ) + .unwrap(); + + state + .db + .create_repo(&seed_repo(&owner_did, "withhold")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "withhold") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + // Withheld secret CID under a walk error → 404, no leak. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "walk error must not serve the withheld blob" + ); + assert!( + !body.contains("TOP SECRET"), + "walk-error 404 must not leak the secret" + ); + + // The PUBLIC blob in the same repo is also 404: the walk error fails closed + // by skipping the whole repo, not by serving. Without the fail-closed arm + // this would serve 200, so this assertion is the load-bearing discriminator. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "walk error fails closed: repo skipped, even the public blob is not served" + ); + } + + /// #173 review (F2): the commit/tag reachability walk must FAIL CLOSED on a git + /// error, exactly like the blob/tree walk. A ref pointing at a nonexistent object + /// makes `rev-list --all` fail, so `reachable_commit_tag_oids` returns Err, which + /// the handler's shared `Ok(Err) => continue` arm turns into a repo skip. The + /// load-bearing discriminator is that the PUBLIC commit is ALSO 404: if the arm + /// fail-OPENed (served on error) it would 200. Drives the commit/tag branch of + /// the shared fail-closed arm specifically (the sibling test covers blob/tree). + #[sqlx::test] + async fn ipfs_cid_commit_tag_walk_error_fails_closed(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["cterr"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("cterr.git"); + // A reachable commit CID — would serve 200 if the walk succeeded. + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + + // A ref to a NONEXISTENT object: `git rev-list --all` fails ("bad object"), + // so reachable_commit_tag_oids bails → the walk arm skips the repo. + std::fs::write( + bare.join("refs/heads/broken"), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n", + ) + .unwrap(); + + state + .db + .create_repo(&seed_repo(&owner_did, "cterr")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "cterr") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // Fail-closed: a walk error skips the repo, so even the otherwise-reachable + // public commit is 404 (not served). A fail-OPEN arm would 200 here. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a commit/tag walk error must fail closed (repo skipped), never serve" + ); + } + + /// #126: a dangling blob (written via `git hash-object -w`, never referenced + /// by any commit/tree) must 404 through `GET /ipfs/{cid}` under path-scoped + /// rules — for anon AND the owner. The pre-#126 deny-set was fail-open by + /// construction: dangling oids were absent from the reachable enumeration + /// and thus absent from the deny-set, so the handler served 200. The + /// allowed-set is fail-closed: dangling oids are absent from the reachable + /// allowed-set, so the handler 404s (per team memory: the owner shift to + /// 404 is the accepted fail-closed default — owners can still + /// `git cat-file` directly). + #[sqlx::test] + async fn ipfs_cid_dangling_blob_fails_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + // Seed a normal repo with `secret/b.txt` reachable from HEAD, so the + // path-scoped rule has something to match — without this the rule has + // no anchor and we'd be testing nothing. + let _fx = seed_cid_repos(&slug, &short, &["dangling"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangling.git"); + + // Write a dangling blob: `git hash-object -w --stdin` adds it to the + // object DB but nothing references it, so the reachable walk never + // enumerates it. + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + let stdin = child.stdin.as_mut().expect("stdin"); + stdin.write_all(b"DANGLING SECRET\n").expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!( + out.status.success(), + "git hash-object: {}", + String::from_utf8_lossy(&out.stderr) + ); + let dangling_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + // Sanity: must be a 64-hex sha256 oid, since the repo is sha256-format. + assert_eq!( + dangling_oid.len(), + 64, + "expected sha256 oid: {dangling_oid}" + ); + // Record the pin so oid_for_cid resolves it — the 404 must then come from + // the allowed-set gate excluding the dangling oid, not from a table miss. + let dangling_cid = pin_cid_for(&bare, &dangling_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangling")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangling") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-blob allowed-set gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + // anon: the dangling blob is absent from the reachable allowed-set → + // 404, no leak. Pre-#126 (deny-set) would serve 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&dangling_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling blob must 404 under path-scoped rules" + ); + assert!( + !body.contains("DANGLING SECRET"), + "404 body must not leak the dangling content" + ); + + // owner (signed): same 404. The dangling blob has no path, so it's + // never visibility-checked → never in the allowed set, even for the + // owner. This is the accepted fail-closed shift documented in the PR. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &dangling_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "owner also 404s on dangling blobs under path-scoped rules (fail-closed default)" + ); + assert!(!body.contains("DANGLING SECRET")); + } + + /// #135: a DANGLING tree (in the ODB, referenced by no commit) 404s under + /// path-scoped rules for anon AND owner — the reachable-only allowed-tree-set + /// never enumerates it. Handler-level companion to the helper test + /// `allowed_tree_set_excludes_dangling_tree`, proving the `get_by_cid` tree arm + /// (memo insert + `!in_allowed` continue) fails closed on the dangling case. + #[sqlx::test] + async fn ipfs_cid_dangling_tree_fails_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["dangtree"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangtree.git"); + + // Dangling tree via `git mktree`: a UNIQUE entry name so its oid is + // content-distinct from every reachable tree (a content-identical tree would + // dedup to a reachable oid — that is T2, not danglingness). + let mut child = std::process::Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("spawn git mktree"); + { + use std::io::Write; + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {}\tdangling-only-unreferenced.txt", + fx.secret_oid + ) + .unwrap(); + } + let out = child.wait_with_output().expect("mktree output"); + assert!( + out.status.success(), + "git mktree: {}", + String::from_utf8_lossy(&out.stderr) + ); + let dangling_tree_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + assert_eq!(dangling_tree_oid.len(), 64, "expected sha256 oid"); + // Record the pin so the 404 is the allowed-tree-set gate excluding the + // dangling tree, not a table miss. + let dangling_cid = pin_cid_for(&bare, &dangling_tree_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangtree")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangtree") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + for req in [cid_anon(&dangling_cid), cid_signed(&owner, &dangling_cid)] { + let (st, _) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling tree must 404 under path-scoped rules (anon + owner)" + ); + } + } + + /// #173 (F1): a QUARANTINED repo must not serve a pinned object by CID, to anon + /// OR to the mirror's own owner — quarantine is "hidden from serve/clone/listings, + /// owner included" (authorize_repo_read / feed_quarantined_mirror_withheld_from_owner). + /// The repo is PUBLIC with no path-scoped rule, so the "/" visibility gate ALLOWS + /// it and quarantine is the sole possible denier: RED before the fix (the loop + /// never checks quarantine → serves 200 + bytes), GREEN after the quarantine skip. + /// The owner-signed 404 is the load-bearing negative — a visibility-only gate + /// would Allow the owner and miss this. + #[sqlx::test] + async fn ipfs_cid_quarantined_repo_withheld_from_anon_and_owner(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["quar"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("quar.git"); + // Pin a ROOT-readable object (public/a.txt) — no path-scoped rule, so only + // quarantine can deny it. + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "quar")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "quar") + .await + .unwrap() + .unwrap(); + + // Baseline: before quarantine the object serves 200 (proves the CID resolves + // and the object is otherwise servable, so the 404 below is quarantine's doing). + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "public root object serves before quarantine" + ); + assert!(body.contains("public bytes"), "baseline serves the content"); + + // Quarantine it. + state + .db + .set_repo_quarantine(&rec.id, true) + .await + .expect("quarantine"); + + // anon AND owner-signed must both 404 with no content leak. + for req in [cid_anon(&public_cid), cid_signed(&owner, &public_cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "quarantined repo must not serve by CID (anon + owner)" + ); + assert!( + !body.contains("public bytes"), + "404 body must not leak quarantined content" + ); + } + } + + /// #173 (F2): a DANGLING commit or annotated tag (in the ODB, referenced by no + /// ref) must 404 under path-scoped rules for anon AND owner. The resolver proved + /// reachability only for blobs/trees, so a dangling commit/tag fell through to + /// serve, leaking its message/metadata. RED before the fix (serves 200 + + /// sentinel), GREEN after (the reachable commit/tag set excludes them). The + /// reachable-commit/tag serve path is covered by + /// ipfs_cid_gate_withholds_blob_from_unauthorized (commit + annotated tag → 200). + #[sqlx::test] + async fn ipfs_cid_dangling_commit_and_tag_fail_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["dangct"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangct.git"); + + // Run a git plumbing command that reads from stdin and prints an oid. + let oid_from_stdin = |args: &[&str], input: &[u8]| -> String { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args(args) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn git"); + child.stdin.as_mut().unwrap().write_all(input).unwrap(); + let out = child.wait_with_output().expect("git output"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Dangling commit: commit-tree with a sentinel message, NO ref update. + let dangling_commit_oid = oid_from_stdin( + &["commit-tree", &fx.root_tree_oid], + b"DANGLING COMMIT SECRET\n", + ); + assert_eq!(dangling_commit_oid.len(), 64, "expected sha256 commit oid"); + // Dangling annotated tag: mktag of the dangling commit, NO ref. + let tag_body = format!( + "object {dangling_commit_oid}\ntype commit\ntag dang\ntagger t 0 +0000\n\nDANGLING TAG SECRET\n" + ); + let dangling_tag_oid = oid_from_stdin(&["mktag"], tag_body.as_bytes()); + assert_eq!(dangling_tag_oid.len(), 64, "expected sha256 tag oid"); + + let commit_cid = pin_cid_for(&bare, &dangling_commit_oid, &state.db).await; + let tag_cid = pin_cid_for(&bare, &dangling_tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangct")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangct") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-object gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + for (cid, sentinel) in [ + (&commit_cid, "DANGLING COMMIT SECRET"), + (&tag_cid, "DANGLING TAG SECRET"), + ] { + for req in [cid_anon(cid), cid_signed(&owner, cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling commit/tag must 404 under path-scoped rules (anon + owner)" + ); + assert!( + !body.contains(sentinel), + "404 body must not leak the dangling message: {sentinel}" + ); + } + } + } + + /// #173 review (F2 hardening): a REACHABLE commit must still serve under a + /// path-scoped rule even when the repo carries a pushable non-commit ref (an + /// annotated tag of a tree, accepted by receive-pack). `reachable_commit_tag_oids` + /// must NOT route through `assert_all_refs_are_commits` (which bails on such a + /// ref and would fail-closed 404 every reachable commit/tag CID in the repo). + /// RED before the decoupling (the guard bails → 404), GREEN after. + #[sqlx::test] + async fn ipfs_cid_reachable_commit_served_despite_non_commit_ref(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["weirdref"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("weirdref.git"); + + // A pushable non-commit ref: an annotated tag pointing at a TREE. `git tag -a` + // in the bare repo creates refs/tags/treetag -> tag object -> tree, which + // peels to a non-commit and makes assert_all_refs_are_commits bail. + let out = std::process::Command::new("git") + .args([ + "tag", + "-a", + "treetag", + &fx.root_tree_oid, + "-m", + "tag of a tree", + ]) + .current_dir(&bare) + .output() + .expect("git tag -a"); + assert!( + out.status.success(), + "git tag -a: {}", + String::from_utf8_lossy(&out.stderr) + ); + + // Pin the REACHABLE root commit. + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "weirdref")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "weirdref") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // The reachable commit must still serve — the non-commit ref must not + // fail-closed the whole repo's commit/tag CID retrieval. + let resp = cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(); + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "a reachable commit must serve despite a pushable non-commit ref in the repo" + ); + assert_eq!( + served_hash.as_deref(), + Some(fx.commit_oid.as_str()), + "the served object is the reachable root commit" + ); + } + + /// #173 review (F-F): an annotated tag pointing at a TREE is pushable through + /// receive-pack, and the TREE allowed-set path + /// (`allowed_tree_set_for_caller` -> `tree_paths` -> `reachable_commits`) runs + /// `assert_all_refs_are_commits`, which bails on that ref and fail-closes the + /// whole repo — 404-ing EVERY tree CID (root + public subtrees) for its owner + /// and readers, not just the offending tag. The tree allowed-set feeds ONLY the + /// CID gate (absence = fail-closed 404), so `tree_paths` uses the lenient + /// reachable-commit enumeration: commit-reachable trees still serve, while a + /// tree reachable only via such a tag stays excluded. `blob_paths` keeps the + /// strict guard (it also feeds serve/replication, where a miss under-withholds). + /// RED before the decoupling (whole-repo bail -> 404 on the root/public tree), + /// GREEN after; the withheld-subtree 404 is the load-bearing must-not. + #[sqlx::test] + async fn ipfs_cid_tree_served_despite_non_commit_ref(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["treeweird"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("treeweird.git"); + + // Pushable non-commit ref: an annotated tag pointing at the ROOT TREE. + let out = std::process::Command::new("git") + .args([ + "tag", + "-a", + "treetag", + &fx.root_tree_oid, + "-m", + "tag of a tree", + ]) + .current_dir(&bare) + .output() + .expect("git tag -a"); + assert!( + out.status.success(), + "git tag -a: {}", + String::from_utf8_lossy(&out.stderr) + ); + + // Pin the reachable root tree and public subtree (both at ALLOWED paths), + // plus the secret subtree (a DENIED path — the fail-closed negative). + let root_tree_cid = pin_cid_for(&bare, &fx.root_tree_oid, &state.db).await; + let public_tree_cid = pin_cid_for(&bare, &fx.public_tree_oid, &state.db).await; + let secret_tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "treeweird")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "treeweird") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-object tree gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // Reachable trees at ALLOWED paths must still serve despite the tag-of-tree. + for (cid, want_oid, label) in [ + (&root_tree_cid, &fx.root_tree_oid, "root tree"), + (&public_tree_cid, &fx.public_tree_oid, "public subtree"), + ] { + let resp = cid_router(&state).oneshot(cid_anon(cid)).await.unwrap(); + let served = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "{label} CID must serve despite a pushable tag-of-tree in the repo" + ); + assert_eq!( + served.as_deref(), + Some(want_oid.as_str()), + "{label}: the served object is the reachable tree" + ); + } + + // Fail-closed preserved: the DENIED subtree's CID is still withheld — the + // lenient walk must not under-withhold a path the caller cannot read. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a withheld subtree's tree CID stays 404 (lenient walk must not under-withhold)" + ); + } + + /// #173 review (F2 hardening): the INNER tag object of a nested tag-of-a-tag is + /// reachable (via the outer ref tag) and pinnable, so its CID must serve under a + /// path rule. `reachable_commit_tag_oids` peels tag chains to include it. RED + /// before the peel loop (the inner tag is not a ref tip and rev-list dereferences + /// to the commit, so it is absent → 404), GREEN after. + #[sqlx::test] + async fn ipfs_cid_nested_tag_inner_object_served(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["nested"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("nested.git"); + + let git_stdin = |args: &[&str], input: &[u8]| -> String { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args(args) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn git"); + child.stdin.as_mut().unwrap().write_all(input).unwrap(); + let out = child.wait_with_output().expect("git output"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Inner annotated tag of the reachable commit (no ref of its own). + let inner_body = format!( + "object {}\ntype commit\ntag inner\ntagger t 0 +0000\n\ninner\n", + fx.commit_oid + ); + let inner_tag_oid = git_stdin(&["mktag"], inner_body.as_bytes()); + // Outer annotated tag of the inner tag, then a ref to the outer tag. The + // inner tag is reachable only THROUGH the outer, not as a ref tip. + let outer_body = format!( + "object {inner_tag_oid}\ntype tag\ntag outer\ntagger t 0 +0000\n\nouter\n" + ); + let outer_tag_oid = git_stdin(&["mktag"], outer_body.as_bytes()); + let out = std::process::Command::new("git") + .args(["update-ref", "refs/tags/nested", &outer_tag_oid]) + .current_dir(&bare) + .output() + .expect("update-ref"); + assert!( + out.status.success(), + "update-ref: {}", + String::from_utf8_lossy(&out.stderr) + ); + + let inner_cid = pin_cid_for(&bare, &inner_tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "nested")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "nested") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&inner_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "the inner tag of a nested tag-of-a-tag is reachable and must serve" + ); + } + + /// #135: with NO path-scoped rule the per-object gate is skipped, so a tree CID + /// is served (the `"/"` gate is the whole story). Guards against over-gating + /// trees — the tree analog of the blob skip-walk branch. + #[sqlx::test] + async fn ipfs_cid_tree_served_when_no_path_scoped_rule(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["nopathrule"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("nopathrule.git"); + let tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + // Public repo, no visibility rules → has_path_scoped_rule is false. + state + .db + .create_repo(&seed_repo(&owner_did, "nopathrule")) + .await + .expect("seed repo"); + + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_anon(&tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "tree served to anon when no path-scoped rule exists" + ); + assert!( + bytes_contain(&body, b"b.txt"), + "served tree carries its child structure" + ); + } + + /// #173 (Fix 1): the pinned_cids lookup must use the canonical base32 CID, not + /// the raw request spelling. A pin is stored under `cid.to_string()` (canonical + /// base32); a request carrying the SAME CID re-encoded to a different multibase + /// (base58btc) parses and passes the sha2-256 check but, on the pre-fix handler, + /// misses the lookup key → false 404. Public repo, no path-scoped rule, so no + /// walk — this isolates the lookup-key canonicalization. + #[sqlx::test] + async fn ipfs_alt_encoding_cid_resolves(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["altenc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("altenc.git"); + // Canonical base32 CID as stored by the pin path. + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + // Public repo, no visibility rules (no path-scoped walk). + state + .db + .create_repo(&seed_repo(&owner_did, "altenc")) + .await + .expect("seed repo"); + + // Re-encode the SAME CID to base58btc — a different, equally-valid spelling + // that is NOT the stored key. The `cid` crate re-exports `multibase`. + let alt = public_cid + .parse::>() + .unwrap() + .to_string_of_base(cid::multibase::Base::Base58Btc) + .unwrap(); + assert_ne!(alt, public_cid, "alt encoding must differ from canonical"); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&alt)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "alt-multibase spelling of a pinned CID must resolve (canonicalized lookup)" + ); + assert!( + body.contains("public bytes"), + "resolved object serves its content" + ); + } + + /// #173 (Fix 2a, db-level): `oids_for_cid` returns EVERY oid recorded under a + /// CID, not an arbitrary one. `record_pinned_cid` is unique on the git oid and + /// non-unique on cid, so two distinct oids can share one content-CID. Old + /// `oid_for_cid` did `LIMIT 1`; the new plural method must surface both. + #[sqlx::test] + async fn oids_for_cid_returns_all_duplicates(pool: PgPool) { + let state = test_state(pool).await; + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"shared content cid").to_string(); + let oid_a = "a".repeat(64); + let oid_b = "b".repeat(64); + state + .db + .record_pinned_cid(&oid_a, &cid, None) + .await + .unwrap(); + state + .db + .record_pinned_cid(&oid_b, &cid, None) + .await + .unwrap(); + + let mut oids = state.db.oids_for_cid(&cid).await.unwrap(); + oids.sort(); + assert_eq!( + oids, + vec![oid_a, oid_b], + "oids_for_cid must return every oid recorded under the shared CID" + ); + } + + /// #173 (Fix 2b, handler-level): when two oids collide on one CID and the + /// first-recorded is absent from every repo while the second is a readable + /// public object, the handler must try both and serve the readable one. The + /// pre-fix handler resolved a single oid (LIMIT 1 → first-inserted for equal + /// keys) and 404'd. Ordering caveat: this relies on `oids_for_cid` returning + /// the absent oid before the readable one (heap/insert order for equal keys); + /// if that ordering ever changes, `oids_for_cid_returns_all_duplicates` remains + /// the load-bearing, deterministic driver for Fix 2. + #[sqlx::test] + async fn ipfs_cid_collision_serves_readable_duplicate(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["collision"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("collision.git"); + + // A GENUINE content collision: the shared CID is the readable object's REAL + // content CID, and a second (absent) oid is recorded under the SAME cid. The + // handler must try every oid and serve the one whose bytes hash to the CID. + // (F2, #173: the served bytes must match the requested content address, so the + // shared cid has to be the object's real cid — an arbitrary seed would now be + // withheld by the integrity check as an unverifiable provider-CID-style row.) + let (_ty, raw) = crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .unwrap(); + let shared_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + let absent_oid = "c".repeat(64); + state + .db + .record_pinned_cid(&absent_oid, &shared_cid, None) + .await + .expect("record absent oid first"); + state + .db + .record_pinned_cid(&fx.public_oid, &shared_cid, None) + .await + .expect("record readable oid second"); + + // Public repo, no rules → the readable public object is served if reached. + state + .db + .create_repo(&seed_repo(&owner_did, "collision")) + .await + .expect("seed repo"); - // anon → repo-level deny → 404, no content leaked. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&blob_cid)) + .oneshot(cid_anon(&shared_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "handler must try every oid under the CID and serve the readable duplicate" + ); + assert!( + body.contains("public bytes"), + "the readable duplicate's content is served" + ); + } + + /// #173 (Fix 3/F3, INV-10): the expensive legacy fan-out is rate-limited per + /// source IP. A valid tree CID makes the object-type pre-check pass, so each + /// repeat request pays a fresh walk (request-scoped memo only) — unbounded + /// amplification. Since #173-F3 (jatmn) the source charge sits on the LEGACY + /// PROBE (`acquire` + `cat-file`), which precedes the walk, so every legacy + /// candidate is charged to the non-farmable source IP from the first probe; a + /// second identical request from the same IP is shed with 429, but a targeted + /// PROVENANCE fetch (no scan) and a request from a different IP are unaffected. + /// The limiter is sized to admit one full scan of the two seeded repos (2 probes) + /// so the first request serves; the repeat then finds the bucket spent. + #[sqlx::test] + async fn ipfs_walk_rate_limited_per_source(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + + let mut state = test_state(pool).await; + // The scan probes both seeded repos (walklimit + walkpublic) per request, so + // size the per-IP budget to admit exactly one full scan (2 probes). A repeat + // scan from the same IP then finds the bucket spent. Keyed on the rightmost + // X-Forwarded-For hop so the test can choose a source IP under `oneshot`. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["walklimit"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("walklimit.git"); + // The tree CID drives a path-scoped walk (the load-bearing amplification + // surface). The reader is allowed under /secret so the walk returns 200. + let secret_tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + // Oldest `updated_at` → `list_all_repos` (ORDER BY updated_at DESC) probes + // this serving repo LAST, so a scan deterministically charges the walk-free + // `walkpublic` miss first then this serve: exactly 2 probes per scan. + let mut walklimit = seed_repo(&owner_did, "walklimit"); + walklimit.updated_at = chrono::Utc::now() - chrono::Duration::seconds(60); + state.db.create_repo(&walklimit).await.expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "walklimit") + .await + .unwrap() + .unwrap(); + // Mode B path rule over /secret with the reader allowed → the reader's + // secret-tree fetch runs the allowed-tree walk and returns 200. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + + // The MUST-NOT object must be a genuinely CHEAP fetch: an object served + // from a repo with NO path-scoped rule takes the no-walk path, so the WALK + // brake never rate-limits it. It has to live in a repo that carries no path + // rule AND whose object graph does not overlap `walklimit` (a blob shared + // with the path-scoped repo would still walk there), so we seed a second bare + // repo with UNIQUE content. `acquire(owner, "walkpublic")` resolves to + // `/tmp//walkpublic.git`. This copy is PROVENANCED (`pin_cid_for_repo`) + // so it resolves straight to its repo and skips the legacy probe brake: the + // point here is the WALK brake, and post-#173-F3 a walk-free LEGACY fetch is + // itself source-charged at the probe, so a legacy pin would (correctly) be + // shed from the exhausted IP and no longer isolate the walk brake. + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("walkpublic.git"); + { + use std::process::Command; + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let src = std::env::temp_dir().join(format!("gl-cid-pub-{short}")); + let _ = std::fs::remove_dir_all(&src); + std::fs::create_dir_all(&src).unwrap(); + std::fs::write(src.join("cheap.txt"), b"cheap public bytes\n").unwrap(); + run(&["init", "-q", "--object-format=sha256"], &src); + run(&["config", "user.email", "t@t"], &src); + run(&["config", "user.name", "t"], &src); + run(&["add", "."], &src); + run(&["commit", "-qm", "cheap"], &src); + let _ = std::fs::remove_dir_all(&pub_bare); + run( + &[ + "clone", + "--bare", + "-q", + src.to_str().unwrap(), + pub_bare.to_str().unwrap(), + ], + &src, + ); + let _ = std::fs::remove_dir_all(&src); + } + let cheap_oid = { + use std::process::Command; + let out = Command::new("git") + .args(["rev-parse", "HEAD:cheap.txt"]) + .current_dir(&pub_bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse cheap.txt"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + // Public repo, NO visibility rules → the cheap object takes the no-walk path. + state + .db + .create_repo(&seed_repo(&owner_did, "walkpublic")) + .await + .expect("seed public repo"); + let pub_rec = state + .db + .get_repo(&owner_did, "walkpublic") + .await + .unwrap() + .unwrap(); + let public_cid = pin_cid_for_repo(&pub_bare, &cheap_oid, &state.db, &pub_rec.id).await; + + // 1st legacy scan from 1.2.3.4 → 200 (its two probes fit the budget; the + // walk ran, reader allowed). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "anon denied at a private repo's / gate" + StatusCode::OK, + "1st legacy scan from a source IP is served" ); - assert!(!body.contains("public bytes"), "404 must not leak content"); - // owner-signed → 200. + // 2nd identical scan from the SAME IP → 429 (per-IP probe budget spent). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "2nd legacy scan from the same source IP is shed with 429" + ); + + // MUST-NOT: a targeted PROVENANCE fetch (no scan, no probe brake) from the + // SAME limited IP, even after the 429, is served: the brake is on the legacy + // scan, not the route. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_signed(&owner, &blob_cid)) + .oneshot(cid_signed_xff(&reader, &public_cid, "1.2.3.4")) .await .unwrap(), ) @@ -2398,109 +5279,156 @@ mod tests { assert_eq!( st, StatusCode::OK, - "owner reads their private repo's object" + "a provenance (non-scan) fetch is never rate-limited, even from the exhausted IP" + ); + assert!( + body.contains("cheap public bytes"), + "the cheap fetch serves content" + ); + + // PER-SOURCE isolation: the same tree-CID scan from a DIFFERENT IP → 200. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "5.6.7.8")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "one source's exhaustion must not shed another source's walk" ); - assert!(body.contains("public bytes"), "owner gets the content"); } - /// Fail-closed walk-error arm: if `withheld_blob_oids` errors (here, a ref - /// pointing at a non-tree-ish blob, which `git ls-tree -r` cannot traverse — - /// the same induction as `visibility_pack::fails_closed_when_a_ref_cannot_be_traversed`), - /// the handler skips the whole repo rather than serving. Asserts no leak of the - /// withheld blob AND that even the *public* blob in that repo is withheld — the - /// latter distinguishes fail-closed-skip from normal per-blob withholding and - /// would serve 200 if the error arm wrongly proceeded. + /// #173 review (F-C): a SKIPPED legacy candidate (a walk-and-deny denier, OR a + /// probe-throttled repo since #173-F3) must not end the whole request: the scan + /// keeps going so a later walk-free copy still serves, and a spent probe budget is + /// a clean 429, never a false 404/503. Otherwise a public CID would 404/429 solely + /// because a newer path-scoped duplicate sorts ahead of an older no-rule copy under + /// `updated_at DESC`. Two same-oid legacy copies: a NEWER `/secret`-scoped denier + /// and an OLDER no-rule public copy. + /// + /// Two requests from the SAME IP, budget = 2 (one full scan of both copies): + /// req1 probes the denier (charged), its allowed-blob walk denies anon → skip and + /// keep scanning, then probes+serves the walk-free public copy → 200. That proves + /// the denier skip is non-fatal (`continue`, not `break`). req2 from the same IP + /// finds the probe budget spent, so the denier's probe throttles → skip-continue, + /// the public copy's probe throttles too → nothing servable → a clean 429 (not a + /// truncation 503 nor a false 404), proving the throttle is likewise non-fatal but + /// correctly shed. RED before `continue` (a `break` on the skipped denier 404s + /// req1 outright). #[sqlx::test] - async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { + async fn ipfs_walk_quota_skips_denier_and_serves_public_copy(pool: PgPool) { use crate::db::VisibilityMode; + use chrono::Utc; use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); - let state = test_state(pool).await; - - let fx = seed_cid_repos(&slug, &short, &["withhold"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - let public_cid = cid_for_oid(&fx.public_oid); - - // Force the withheld walk to fail closed: a ref pointing at a blob (not - // tree-ish) makes `git ls-tree -r` error, which `withheld_blob_oids` - // propagates as Err → the handler's `Ok(Err)` arm skips the repo. - let bare = std::path::PathBuf::from("/tmp") + let mut state = test_state(pool).await; + // Budget = one full two-repo scan (2 probes), keyed on the rightmost XFF hop + // so `oneshot` can choose a source IP (no socket peer). A repeat scan from the + // same IP then finds the budget spent. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + // Identical secret-blob content in both bare clones → one CID resolves to + // `secret_oid` in each. A NEWER path-scoped denier (walk-and-deny anon) and an + // OLDER no-rule public copy (walk-free serve). + let fx = seed_cid_repos(&slug, &short, &["scopeddenier", "publiccopy"]); + let denier_bare = std::path::PathBuf::from("/tmp") .join(&slug) - .join("withhold.git"); - std::fs::write( - bare.join("refs/heads/blobref"), - format!("{}\n", fx.secret_oid), - ) - .unwrap(); - + .join("scopeddenier.git"); + let secret_cid = pin_cid_for(&denier_bare, &fx.secret_oid, &state.db).await; + + // Newer denier: public at "/", `/secret/**` Mode B empty readers → an anon + // blob fetch clears "/", runs the allowed-blob walk, is denied → continue. + let mut denier = seed_repo(&owner_did, "scopeddenier"); + denier.updated_at = Utc::now(); + state.db.create_repo(&denier).await.expect("seed denier"); state .db - .create_repo(&seed_repo(&owner_did, "withhold")) - .await - .expect("seed repo"); - let rec = state - .db - .get_repo(&owner_did, "withhold") + .set_visibility_rule(&denier.id, "/secret/**", VisibilityMode::B, &[], &owner_did) .await - .unwrap() - .unwrap(); + .expect("path rule"); + + // Older public copy — NO rule → the secret blob serves via the no-walk path. + let mut public = seed_repo(&owner_did, "publiccopy"); + public.updated_at = Utc::now() - chrono::Duration::seconds(60); state .db - .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .create_repo(&public) .await - .expect("deny rule"); + .expect("seed public copy"); - // Withheld secret CID under a walk error → 404, no leak. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&secret_cid)) - .await - .unwrap(), - ) - .await; + // req1 from 1.2.3.4: the denier is skipped (walk denies anon) and the scan + // keeps going to serve the older walk-free public copy. Both probes fit the + // budget, so this leaves the IP bucket spent. + let resp = cid_router(&state) + .oneshot(cid_anon_xff(&secret_cid, "1.2.3.4")) + .await + .unwrap(); + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error must not serve the withheld blob" + StatusCode::OK, + "a skipped walk-requiring denier must not end the scan: the later walk-free public copy still serves" ); - assert!( - !body.contains("TOP SECRET"), - "walk-error 404 must not leak the secret" + assert_eq!( + served_hash.as_deref(), + Some(fx.secret_oid.as_str()), + "the served object is the secret blob from the no-rule public copy" ); - // The PUBLIC blob in the same repo is also 404: the walk error fails closed - // by skipping the whole repo, not by serving. Without the fail-closed arm - // this would serve 200, so this assertion is the load-bearing discriminator. + // req2 from the SAME exhausted IP: every legacy probe is now throttled. The + // throttle is non-fatal (skip and keep scanning), but nothing is servable, so + // it resolves to a clean 429, not a truncation 503, not a false 404. let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&public_cid)) + .oneshot(cid_anon_xff(&secret_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error fails closed: repo skipped, even the public blob is not served" + StatusCode::TOO_MANY_REQUESTS, + "with the probe budget spent, the repeat legacy scan is shed with a clean 429" ); } - /// #126: a dangling blob (written via `git hash-object -w`, never referenced - /// by any commit/tree) must 404 through `GET /ipfs/{cid}` under path-scoped - /// rules — for anon AND the owner. The pre-#126 deny-set was fail-open by - /// construction: dangling oids were absent from the reachable enumeration - /// and thus absent from the deny-set, so the handler served 200. The - /// allowed-set is fail-closed: dangling oids are absent from the reachable - /// allowed-set, so the handler 404s (per team memory: the owner shift to - /// 404 is the accepted fail-closed default — owners can still - /// `git cat-file` directly). + /// INV-10 amplification bound: a single `GET /ipfs/{cid}` must not fan out an + /// unbounded number of full-history walks. The per-request `ipfs_rate_limiter` + /// check only brakes REPEAT requests (it fires once per request); within one + /// request the same object can exist under path-scoped rules in many repos, + /// each paying its own walk. `MAX_HISTORY_WALKS_PER_REQUEST` caps that fan-out. + /// + /// Load-bearing witness (#173, F4): a readable public copy (no path rule → + /// served via the no-walk path, exactly like + /// `ipfs_cid_served_from_public_copy_when_withheld_elsewhere`) is given the + /// OLDEST `updated_at` so `list_all_repos` (ORDER BY updated_at DESC) iterates it + /// LAST. Ahead of it sit `cap + 1` path-scoped deniers, each forcing an + /// allowed-blob walk that denies anon. The cap bounds SPAWNED walks to `cap`, but + /// hitting it must `continue` (skip only the walk-requiring denier), NOT `break` + /// the whole repo loop: the walk-free public copy needs no walk, so it is still + /// reached and served (200, `x-git-hash` = the blob oid). The old `break` + /// wrongly 404'd this publicly-readable content. Reverting `continue`→`break` + /// turns this 200 back into a 404: the RED proof that the loop keeps scanning for + /// a cheap readable copy after the cap. The `cap` walk ceiling still holds — only + /// `cap` walks are spawned across the deniers regardless (the amplification bound + /// is proven separately by `ipfs_walk_cap_still_serves_walk_free_candidate`). #[sqlx::test] - async fn ipfs_cid_dangling_blob_fails_closed_under_path_rules(pool: PgPool) { + async fn ipfs_walk_fanout_capped_per_request(pool: PgPool) { use crate::db::VisibilityMode; + use chrono::Utc; use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); @@ -2509,96 +5437,246 @@ mod tests { let short = owner_did.split(':').next_back().unwrap().to_string(); let state = test_state(pool).await; - // Seed a normal repo with `secret/b.txt` reachable from HEAD, so the - // path-scoped rule has something to match — without this the rule has - // no anchor and we'd be testing nothing. - let _fx = seed_cid_repos(&slug, &short, &["dangling"]); - let bare = std::path::PathBuf::from("/tmp") - .join(&slug) - .join("dangling.git"); + let cap = crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST as usize; - // Write a dangling blob: `git hash-object -w --stdin` adds it to the - // object DB but nothing references it, so the reachable walk never - // enumerates it. - let mut cmd = std::process::Command::new("git"); - cmd.args(["hash-object", "-w", "--stdin"]) - .current_dir(&bare) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()); - let mut child = cmd.spawn().expect("spawn git hash-object"); - { - use std::io::Write; - let stdin = child.stdin.as_mut().expect("stdin"); - stdin.write_all(b"DANGLING SECRET\n").expect("write stdin"); - } - let out = child.wait_with_output().expect("hash-object output"); - assert!( - out.status.success(), - "git hash-object: {}", - String::from_utf8_lossy(&out.stderr) - ); - let dangling_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); - // Sanity: must be a 64-hex sha256 oid, since the repo is sha256-format. - assert_eq!( - dangling_oid.len(), - 64, - "expected sha256 oid: {dangling_oid}" - ); - let dangling_cid = cid_for_oid(&dangling_oid); + // `cap + 1` deniers guarantee the fan-out crosses the ceiling before the + // readable copy (iterated last) is reached. All bare clones share identical + // content, so the one secret-BLOB CID resolves to `secret_oid` in every repo. + let denier_names: Vec = (0..=cap).map(|i| format!("denier{i}")).collect(); + let mut names: Vec<&str> = vec!["readable"]; + names.extend(denier_names.iter().map(|s| s.as_str())); + let fx = seed_cid_repos(&slug, &short, &names); + let readable_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("readable.git"); + // The secret BLOB CID drives the path-scoped allowed-blob walk in every + // denier (the amplification surface) and is served cheaply from the + // no-rule public copy — the proven serve path. + let secret_cid = pin_cid_for(&readable_bare, &fx.secret_oid, &state.db).await; + + // 1) Readable public copy — OLDEST updated_at → iterated LAST. Public with + // NO visibility rule, so the blob serves via the no-walk path. This is + // the copy an uncapped fan-out would eventually reach and serve. + let mut readable = seed_repo(&owner_did, "readable"); + readable.updated_at = Utc::now() - chrono::Duration::seconds(60); state .db - .create_repo(&seed_repo(&owner_did, "dangling")) - .await - .expect("seed repo"); - let rec = state - .db - .get_repo(&owner_did, "dangling") + .create_repo(&readable) + .await + .expect("seed readable copy"); + + // 2) cap+1 deniers with NEWER updated_at → iterated before the copy. Public + // at "/", but a `/secret/**` Mode B rule with an EMPTY reader list, so an + // anon blob fetch clears the "/" gate, runs the allowed-blob walk, and is + // denied (the secret blob is in no one's set) → continue. Each distinct + // repo.id is its own walk (the memo only dedups the same repo). + for name in &denier_names { + let mut denier = seed_repo(&owner_did, name); + denier.updated_at = Utc::now(); + state.db.create_repo(&denier).await.expect("seed denier"); + state + .db + .set_visibility_rule(&denier.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } + + // Anon (no peer, no XFF → the IP brake is skipped, so the walk cap is the + // only thing in play). After the cap, `continue` skips only the + // walk-requiring deniers and keeps scanning, reaching the walk-free public + // copy (iterated last) → served 200. The served object is the secret blob + // from the no-rule public copy, which is legitimately public THERE. + let resp = cid_router(&state) + .oneshot(cid_anon(&secret_cid)) .await - .unwrap() .unwrap(); - // Path-scoped rule triggers the per-blob allowed-set gate (KTD4). + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "hitting the walk cap must skip only the walk-requiring candidate, not abandon the walk-free readable copy" + ); + assert_eq!( + served_hash.as_deref(), + Some(fx.secret_oid.as_str()), + "the served object is the blob from the no-rule public copy reached after the cap" + ); + } + + /// Multi-oid companion to `ipfs_walk_fanout_capped_per_request`: exercises the + /// outer oid loop and proves the per-request walk budget PERSISTS across oid + /// candidates, so a commit/tag candidate cannot re-open the fan-out. Since #173 + /// (F2) a `commit`/`tag` under a path-scoped rule is itself walk-gated (its + /// reachability is proven by a `rev-list` walk via `reachable_commit_tag_oids`), + /// so it is NOT walk-free — it draws from the same budget as the blob/tree walks. + /// + /// One CID → TWO oids (the non-unique cid index, #173): a withheld `/secret` + /// blob (walk-triggering, denied to anon in every denier) recorded FIRST so a + /// seq scan tries it first and burns the whole walk budget across the deniers; + /// the reachable root commit is second. Because the budget is already spent, the + /// commit candidate's reachability walk is also capped in every denier, so the + /// request 404s — proving commit/tag walks (F2) respect the fan-out ceiling and + /// cannot be used to bypass it (R6/F3). A reachable commit served with budget to + /// spare is covered by `ipfs_cid_gate_withholds_blob_from_unauthorized`. The + /// withheld blob must not leak. Since #173 F2 a scan the walk cap truncated + /// returns 503 (absence unproven), not the old opaque 404. + #[sqlx::test] + async fn ipfs_walk_commit_tag_candidate_respects_the_walk_cap(pool: PgPool) { + use crate::db::VisibilityMode; + use chrono::Utc; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let cap = crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST as usize; + + // cap+1 path-scoped deniers, all carrying identical content (same oids). + let denier_names: Vec = (0..=cap).map(|i| format!("m{i}")).collect(); + let names: Vec<&str> = denier_names.iter().map(|s| s.as_str()).collect(); + let fx = seed_cid_repos(&slug, &short, &names); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("m0.git"); + + // ONE cid → TWO oids. The withheld blob is recorded first (seq scan lists it + // first → tried first → burns the budget); the reachable commit is second. + let multi_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; state .db - .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .record_pinned_cid(&fx.commit_oid, &multi_cid, None) .await - .expect("deny rule"); + .expect("co-locate the commit oid under the same cid"); - // anon: the dangling blob is absent from the reachable allowed-set → - // 404, no leak. Pre-#126 (deny-set) would serve 200. + for name in &denier_names { + let mut d = seed_repo(&owner_did, name); + d.updated_at = Utc::now(); + state.db.create_repo(&d).await.expect("seed denier"); + state + .db + .set_visibility_rule(&d.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } + + // Anon: the blob candidate is denied in every denier (a walk each, spending + // the budget); the commit candidate's reachability walk is then also capped + // in every denier — so no candidate is served AND the walk cap truncated the + // scan, leaving absence unproven → 503 (not the old false 404, #173 F2). + // Either way commit/tag walks respect the ceiling and cannot re-open the + // fan-out (R6/F3). The withheld blob must not leak in the body. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&dangling_cid)) + .oneshot(cid_anon(&multi_cid)) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "dangling blob must 404 under path-scoped rules" + StatusCode::SERVICE_UNAVAILABLE, + "a commit/tag reachability walk respects the per-request cap; a truncated scan is 503, not a false 404" ); assert!( - !body.contains("DANGLING SECRET"), - "404 body must not leak the dangling content" + !body.contains("TOP SECRET"), + "the withheld blob must not leak in the truncation response" ); + } - // owner (signed): same 404. The dangling blob has no path, so it's - // never visibility-checked → never in the allowed set, even for the - // owner. This is the accepted fail-closed shift documented in the PR. - let (st, body) = cid_parts( + /// #173 (F3, INV-15): the per-IP quota debits ONE token per expensive legacy + /// candidate, not once per request, so one IP cannot drive an unbounded fan-out. + /// With quota=1 and two path-scoped deniers holding one CID, a SINGLE request is + /// shed at 429: since #173-F3 (jatmn) each legacy PROBE (`acquire` + `cat-file`, + /// which precedes the walk) debits, so the first denier probes+walks+denies on + /// token 1 and the second denier's probe finds no token → 429. (Before F3 the + /// debit sat on the walk; the outcome is unchanged, the charge point moved earlier + /// to also bound walk-free probes.) Defeating the per-candidate debit let one IP + /// drive up to MAX_HISTORY_WALKS_PER_REQUEST × quota expensive ops/hour. + #[sqlx::test] + async fn ipfs_walk_quota_debited_per_walk(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + // Signed but NOT a reader → cleared at "/", denied at /secret → forces a walk. + let stranger = Keypair::generate(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["w0", "w1"]); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("w0.git"); + // The secret BLOB CID forces a path-scoped allowed-blob walk in each denier. + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + + // Two path-scoped deniers (Mode B /secret, empty readers): each forces a + // walk that denies the signed stranger, so ONE request spawns two walks. + for name in ["w0", "w1"] { + let d = seed_repo(&owner_did, name); + state.db.create_repo(&d).await.expect("seed denier"); + state + .db + .set_visibility_rule(&d.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } + + // ONE request, quota 1: walk 1 debits the token, walk 2 has none → 429. + let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_signed(&owner, &dangling_cid)) + .oneshot(cid_signed_xff(&stranger, &secret_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "owner also 404s on dangling blobs under path-scoped rules (fail-closed default)" + StatusCode::TOO_MANY_REQUESTS, + "the second full-history walk in one request must be shed with 429 (per-walk debit)" + ); + } + + /// The periodic cleanup task must sweep the ipfs walk limiter, not only its + /// five siblings. Drives `AppState::sweep_rate_limiters` — the exact method the + /// 300s loop calls — and asserts the ipfs limiter's expired entry is evicted. + /// Dropping `ipfs_rate_limiter.cleanup()` from that method leaves the entry in + /// place (`tracked_keys` stays 1): the RED proof that the sweep covers it. + #[sqlx::test] + async fn sweep_rate_limiters_includes_ipfs_limiter(pool: PgPool) { + let mut state = test_state(pool).await; + // Short window so a single recorded hit is already expired at sweep time. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(5, Duration::from_millis(50)); + + assert!( + state.ipfs_rate_limiter.check("1.2.3.4").await, + "record a hit on the ipfs limiter" + ); + assert_eq!( + state.ipfs_rate_limiter.tracked_keys().await, + 1, + "the source-IP key is tracked before the sweep" + ); + + // Expire the entry (still mapped — cleanup hasn't run), then sweep. + tokio::time::sleep(Duration::from_millis(60)).await; + state.sweep_rate_limiters().await; + + assert_eq!( + state.ipfs_rate_limiter.tracked_keys().await, + 0, + "the periodic sweep must evict the ipfs limiter's expired entries" ); - assert!(!body.contains("DANGLING SECRET")); } // --------------------------------------------------------------------------- diff --git a/crates/gitlawb-node/src/visibility.rs b/crates/gitlawb-node/src/visibility.rs index 56616872..a8d7ddba 100644 --- a/crates/gitlawb-node/src/visibility.rs +++ b/crates/gitlawb-node/src/visibility.rs @@ -437,6 +437,31 @@ mod tests { ); } + // #135 T1: a Mode-B rule on `/secret/**` must DENY the withheld directory's + // OWN path `/secret` (the `path == prefix` arm), not just strict descendants — + // otherwise get_by_cid's tree gate would serve the /secret tree object and leak + // its children. Pins parity with get_tree, which denies the /secret path. + #[test] + fn subtree_rule_denies_the_withheld_directory_itself() { + let reader = "did:key:z6MkReader"; + let rules = [rule("/secret/**", VisibilityMode::B, &[reader])]; + assert_eq!( + visibility_check(&rules, true, OWNER, None, "/secret"), + Decision::Deny, + "anon denied at the withheld directory's OWN path /secret" + ); + assert_eq!( + visibility_check(&rules, true, OWNER, None, "/public"), + Decision::Allow, + "anon allowed at a sibling path outside the withheld subtree" + ); + assert_eq!( + visibility_check(&rules, true, OWNER, Some(reader), "/secret"), + Decision::Allow, + "listed reader allowed at the withheld directory (caller-aware)" + ); + } + // #153 regression: cross-method DID must still be denied even when the // trailing segment collides with a bare owner key. #[test] diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index d4cb64fc..fa7a3f3f 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -33,13 +33,16 @@ pub enum IpfsCmd { cid: String, #[arg(long, default_value = "https://node.gitlawb.com", env = "GITLAWB_NODE")] node: String, + /// Identity directory (default: ~/.gitlawb) + #[arg(long)] + dir: Option, }, } pub async fn run(args: IpfsArgs) -> Result<()> { match args.cmd { IpfsCmd::List { node, dir } => cmd_list(node, dir).await, - IpfsCmd::Get { cid, node } => cmd_get(cid, node).await, + IpfsCmd::Get { cid, node, dir } => cmd_get(cid, node, dir).await, } } @@ -86,11 +89,32 @@ async fn cmd_list(node: String, dir: Option) -> Result<()> { Ok(()) } -async fn cmd_get(cid: String, node: String) -> Result<()> { - let client = NodeClient::new(&node, None); - let path = format!("/ipfs/{cid}"); +async fn cmd_get(cid: String, node: String, dir: Option) -> Result<()> { + // #173 (F5): the resolver now serves path-scoped objects to authorized readers, + // so sign with an available identity like `gl ipfs list` — otherwise an owner or + // listed reader gets the opaque anonymous 404 for content they can read. + // `get_authed` signs when a keypair is present and falls back to unsigned. + // + // An explicit `--dir` is a request to use THAT identity: propagate a + // missing/corrupt-keystore error (like `list`) instead of silently sending an + // anonymous request the authorized reader would see as the node's opaque 404 + // (#173 review). Only the default (no `--dir`) keeps the best-effort unsigned + // fallback, so `get` stays usable for genuinely public content. + let keypair = match dir.as_deref() { + Some(dir) => Some(crate::identity::load_keypair_from_dir(Some(dir))?), + None => crate::identity::load_keypair_from_dir(None).ok(), + }; + let client = NodeClient::new(&node, keypair); + // #173 review (F1): the node now accepts equivalent multibase spellings, + // including base64 CIDs (prefix 'm'), whose alphabet contains '/', '+', '='. + // Interpolating the CID raw would make the client request (and sign) + // `/ipfs//`, which neither matches the single-segment Axum + // route nor points at the intended target. Percent-encode the CID as exactly + // one path segment so the signed and sent target agree and the server's + // `Path` extractor decodes it back to the original CID. + let path = format!("/ipfs/{}", encode_cid_segment(&cid)); let resp = client - .get(&path) + .get_authed(&path) .await .with_context(|| format!("failed to fetch CID {cid} from {node}"))?; @@ -119,6 +143,16 @@ async fn cmd_get(cid: String, node: String) -> Result<()> { Ok(()) } +/// Percent-encode a CID so it occupies exactly one path segment of `/ipfs/`. +/// `urlencoding::encode` escapes every byte outside the RFC 3986 unreserved set +/// (ALPHA / DIGIT / `-._~`), so the base64-CID characters that would otherwise +/// break the single-segment route — `/`, `+`, `=` — are all escaped, and the +/// server's `Path` extractor decodes the result back to the original CID (#173 +/// review, F1). +fn encode_cid_segment(cid: &str) -> String { + urlencoding::encode(cid).into_owned() +} + #[cfg(test)] mod tests { use super::*; @@ -235,4 +269,162 @@ mod tests { m.assert_async().await; } + + /// #173 (F5): `gl ipfs get` must SIGN with an available identity, like + /// `gl ipfs list`, so an owner/reader can retrieve a path-scoped object the node + /// now resolves by CID. RED before the fix: cmd_get ignores the identity dir and + /// sends an unsigned request, so the signature-matching mock is never hit + /// (cmd_get errors on the unmatched 501, and m.assert fails). GREEN after: the + /// signed request carries the RFC 9421 headers and is served 200. + #[tokio::test] + async fn test_cmd_get_signs_when_identity_present() { + let mut server = mockito::Server::new_async().await; + let keystore = seed_keystore(); + + let m = server + .mock("GET", "/ipfs/bafkreitestcid") + .match_header("signature", mockito::Matcher::Any) + .match_header("signature-input", mockito::Matcher::Any) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_header("x-git-hash", "abc123") + .with_body("object bytes") + .create_async() + .await; + + cmd_get( + "bafkreitestcid".to_string(), + server.url(), + Some(keystore.path().to_path_buf()), + ) + .await + .expect("signed get of a resolvable object should succeed"); + + m.assert_async().await; + } + + /// #173 (F5) must-not: a genuine anonymous denial must surface as an error, not + /// be masked as success. With no identity dir the request is unsigned; a 404 + /// from the node must produce an Err mentioning the status. + #[tokio::test] + async fn test_cmd_get_anonymous_denial_is_error() { + let mut server = mockito::Server::new_async().await; + + let m = server + .mock("GET", "/ipfs/bafkreidenied") + .with_status(404) + .with_header("content-type", "text/plain") + .with_body("no git object found") + .create_async() + .await; + + let err = cmd_get("bafkreidenied".to_string(), server.url(), None) + .await + .expect_err("a 404 denial must be an error, not masked success"); + assert!( + err.to_string().contains("404"), + "error should mention the status, got: {err}" + ); + + m.assert_async().await; + } + + /// #173 (INV-8) must-not: the node's new 503 "search incomplete" (the legacy CID + /// scan hit its bound and could not prove absence) must surface as an actionable + /// Err naming the status, NOT be rendered as an empty/"not found" success — a + /// retryable outcome the caller has to see. Mirrors the 404 denial case for the + /// bounded-search response the resolver now emits. + #[tokio::test] + async fn test_cmd_get_search_incomplete_503_is_error() { + let mut server = mockito::Server::new_async().await; + + let m = server + .mock("GET", "/ipfs/bafkreiincomplete") + .with_status(503) + .with_header("content-type", "application/json") + .with_body(r#"{"error":"search_incomplete","message":"CID search incomplete — retry"}"#) + .create_async() + .await; + + let err = cmd_get("bafkreiincomplete".to_string(), server.url(), None) + .await + .expect_err("a 503 incomplete-search must be an error, not masked as not-found"); + assert!( + err.to_string().contains("503"), + "error should mention the status, got: {err}" + ); + + m.assert_async().await; + } + + /// #173 review (F1): a base64 CID (multibase prefix 'm') can contain '/', '+', + /// and '='. The client must percent-encode it into ONE path segment before + /// building and signing `/ipfs/`; otherwise the '/' splits the target so + /// it misses the single-segment Axum route and the signature covers the wrong + /// path. Assert the encoded segment carries no raw '/', '+', or '=', and that + /// it decodes back to the original CID (the server's `Path` extractor performs + /// that same decode). RED with the old raw `format!("/ipfs/{cid}")`: the + /// segment still contains '/'. + #[test] + fn test_encode_cid_segment_escapes_base64_alphabet() { + let cid = "mFoo/Bar+baz=="; + let encoded = encode_cid_segment(cid); + + assert!( + !encoded.contains('/'), + "encoded CID must be a single path segment (no raw '/'), got: {encoded}" + ); + assert!( + !encoded.contains('+'), + "encoded CID must escape '+', got: {encoded}" + ); + assert!( + !encoded.contains('='), + "encoded CID must escape '=', got: {encoded}" + ); + + let decoded = urlencoding::decode(&encoded).expect("encoded CID must decode"); + assert_eq!( + decoded, cid, + "encoding must round-trip back to the original CID" + ); + } + + /// #173 review: `gl ipfs get --dir ` must PROPAGATE a missing/corrupt + /// identity-load error like `gl ipfs list`, not silently fall back to an anonymous + /// request — otherwise an authorized reader pointing `--dir` at a broken keystore + /// gets the node's opaque 404 instead of the actionable key-load error. The + /// unsigned fallback is preserved only when NO `--dir` is given (covered by + /// `test_cmd_get_anonymous_denial_is_error`). RED before the fix (`.ok()` swallows + /// the error, an anonymous request is sent, and the `.expect(0)` mock is hit), + /// GREEN after. + #[tokio::test] + async fn test_cmd_get_explicit_dir_no_identity_errors_without_request() { + let mut server = mockito::Server::new_async().await; + // Empty keystore dir passed explicitly via --dir: no identity.pem present. + let empty = tempfile::TempDir::new().unwrap(); + + // The endpoint must never be hit when an explicit --dir fails to load. + let m = server + .mock("GET", "/ipfs/bafkreitestcid") + .expect(0) + .create_async() + .await; + + let err = cmd_get( + "bafkreitestcid".to_string(), + server.url(), + Some(empty.path().to_path_buf()), + ) + .await + .expect_err("an explicit --dir that fails to load must be an error"); + assert!( + err.to_string().contains("gl identity new") + || err.to_string().contains("no identity found") + || err.to_string().contains("failed to load keypair"), + "error should name the key-load failure, got: {err}" + ); + + m.assert_async().await; + } } From 332a5785ca43e030c3e23e6370f36319a5eb5533 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 17 Jul 2026 09:06:21 -0500 Subject: [PATCH 02/77] fix(node): close grok P1 findings on the #173/#174 IPFS integration P1-A: gate_and_serve held the /ipfs walk permit across a bare repo_store.acquire; wrap it in git_acquire_timeout_secs (mirroring #174's own handler) so a cold/hung Tigris acquire cannot pin the global walk slot. On expiry skip the repo and mark the search truncated (retryable 503, not a false 404). P1-B: the local-IPFS pin path recorded Kubo's provider Hash as pinned_cids.cid; for objects above the block size that is a dag-pb root that does not hash the raw content, so GET /ipfs/{cid} listed then 404'd them under the F2 integrity check. Record the locally-computed raw-content CID, mirroring the pinata twin. --- crates/gitlawb-node/src/api/ipfs.rs | 23 ++++++++++++++++++++--- crates/gitlawb-node/src/ipfs_pin.rs | 12 ++++++++++-- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index a764901b..fe16582b 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -502,9 +502,26 @@ async fn gate_and_serve( } walk.probes += 1; } - let repo_path = match state.repo_store.acquire(&repo.owner_did, &repo.name).await { - Ok(p) => p, - Err(_) => return GateOutcome::Skip, + // Bound the per-repo acquire under `git_acquire_timeout_secs`: this gate runs while + // the /ipfs walk permit is held (F5), so a hung or cold-Tigris acquire would otherwise + // pin the global walk slot for the whole request. On expiry skip the repo (a public + // copy may still serve) and mark the search truncated so a wholly-unserved request + // tails to a retryable 503, never a false 404 (reopened the #174 P1-2 stall vector on + // this path otherwise). + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let repo_path = match tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&repo.owner_did, &repo.name), + ) + .await + { + Ok(Ok(p)) => p, + Ok(Err(_)) => return GateOutcome::Skip, + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs gate; skipping repo"); + walk.truncated = true; + return GateOutcome::Skip; + } }; // Existence probe before any walk (random-CID spray must not trigger a walk on a diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index a4ed8e4d..bdc6e090 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -159,7 +159,15 @@ pub async fn pin_new_objects( // Pin to IPFS match pin_git_object(ipfs_api, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinned_cid(&sha, &cid, Some(repo_id)).await { + // The resolver key (`pinned_cids.cid`) must be the locally-computed + // raw-content CID, never the provider Hash: Kubo returns a dag-pb/UnixFS + // root for objects above its block size, which does not hash the raw + // content, so `GET /ipfs/{provider_cid}` would resolve then fail the F2 + // integrity check (list-then-404). The serve path reads bytes from git and + // verifies them against the requested CID, so the raw CID is the correct + // key. Mirrors the pinata twin, which already records the raw CID. + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); + if let Err(e) = db.record_pinned_cid(&sha, &raw_cid, Some(repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); } // F1 (#173 round 8): also record the first pinner in pin_repo_sources so @@ -167,7 +175,7 @@ pub async fn pin_new_objects( if let Err(e) = db.record_pin_source(&sha, repo_id).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); } - pinned.push((sha, cid)); + pinned.push((sha, raw_cid)); } Ok(_) => {} Err(e) => { From dbadd243b45ef975bce2e05638a398c7830b530e Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 17 Jul 2026 09:18:16 -0500 Subject: [PATCH 03/77] fix(node): bound the /ipfs cat-file probes under the walk permit; backfill pinata provenance P2-E: the object-type probe and the F6 size+read ran bare git cat-file inside spawn_blocking while the /ipfs walk permit was held, so a wedged cat-file could pin the global walk slot for the request's life. Bound both under git_service_timeout_secs; on timeout free the slot (truncated -> retryable 503) and skip the repo. P2-D: the pinata already-pinned branch now backfills NULL first-pinner provenance in lockstep with the ipfs_pin skip branch (consistency; the object was already resolvable via the pin_repo_sources union). --- crates/gitlawb-node/src/api/ipfs.rs | 101 ++++++++++++++++++---------- crates/gitlawb-node/src/pinata.rs | 16 +++++ 2 files changed, 82 insertions(+), 35 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index fe16582b..cb18d52a 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -530,17 +530,34 @@ async fn gate_and_serve( let obj_type = { let rp = repo_path.clone(); let sha = sha256_hex.to_string(); - match tokio::task::spawn_blocking(move || store::object_type(&rp, &sha)).await { - Ok(Ok(Some(t))) => t, - Ok(Ok(None)) => return GateOutcome::Skip, - Ok(Err(e)) => { + // Bound the blocking `git cat-file -t` under `git_service_timeout_secs`: this probe + // runs while the /ipfs walk permit is held, so a wedged cat-file (corrupt pack, NFS + // stall) would otherwise pin the global walk slot for the request's life. On timeout + // free the slot (mark truncated -> retryable 503) and skip the repo. spawn_blocking + // cannot be cancelled, so the child may linger on a blocking-pool thread, but it no + // longer holds the walk permit. + let probe_deadline = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + match tokio::time::timeout( + probe_deadline, + tokio::task::spawn_blocking(move || store::object_type(&rp, &sha)), + ) + .await + { + Ok(Ok(Ok(Some(t)))) => t, + Ok(Ok(Ok(None))) => return GateOutcome::Skip, + Ok(Ok(Err(e))) => { tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); return GateOutcome::Skip; } - Err(e) => { + Ok(Err(e)) => { tracing::warn!(repo = %repo.name, err = %e, "object-type probe task panicked; skipping repo"); return GateOutcome::Skip; } + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "object-type probe timed out under the /ipfs walk permit; skipping repo"); + walk.truncated = true; + return GateOutcome::Skip; + } } }; @@ -670,30 +687,49 @@ async fn gate_and_serve( let read_sha = sha256_hex.to_string(); let read_type = obj_type.clone(); let want_cid = ctx.canonical_cid.to_string(); - let read = tokio::task::spawn_blocking(move || -> ServedRead { - match store::object_size(&read_repo, &read_sha) { - Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), - Ok(Some(_)) => {} - // git ran and reported no such object (or an unparseable size): genuine - // not-found for this candidate. - Ok(None) => return ServedRead::Gone, - // git itself failed to run: an infra failure, not a not-found. - Err(e) => return ServedRead::ReadErr(e.to_string()), + // Bound the blocking size+read+verify under `git_service_timeout_secs` (same rationale + // as the object-type probe): a hung cat-file must not pin the held /ipfs walk permit. + let read_deadline = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let read = tokio::time::timeout( + read_deadline, + tokio::task::spawn_blocking(move || -> ServedRead { + match store::object_size(&read_repo, &read_sha) { + Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), + Ok(Some(_)) => {} + // git ran and reported no such object (or an unparseable size): genuine + // not-found for this candidate. + Ok(None) => return ServedRead::Gone, + // git itself failed to run: an infra failure, not a not-found. + Err(e) => return ServedRead::ReadErr(e.to_string()), + } + let content = match store::read_object_content(&read_repo, &read_sha, &read_type) { + Ok(c) => c, + Err(e) => return ServedRead::ReadErr(e.to_string()), + }; + let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); + if served != want_cid { + return ServedRead::Mismatch(served); + } + ServedRead::Ok(content) + }), + ) + .await; + let served_read = match read { + Ok(Ok(sr)) => sr, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "object read task panicked"); + walk.truncated = true; + return GateOutcome::Skip; } - let content = match store::read_object_content(&read_repo, &read_sha, &read_type) { - Ok(c) => c, - Err(e) => return ServedRead::ReadErr(e.to_string()), - }; - let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); - if served != want_cid { - return ServedRead::Mismatch(served); + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "object read timed out under the /ipfs walk permit; skipping repo"); + walk.truncated = true; + return GateOutcome::Skip; } - ServedRead::Ok(content) - }) - .await; - let content = match read { - Ok(ServedRead::Ok(c)) => c, - Ok(ServedRead::TooLarge(size)) => { + }; + let content = match served_read { + ServedRead::Ok(c) => c, + ServedRead::TooLarge(size) => { tracing::warn!( repo = %repo.name, size, max = max_bytes, "withholding object: exceeds the served-object size cap (F6)" @@ -702,15 +738,15 @@ async fn gate_and_serve( note_oversize_reject(); return GateOutcome::Skip; } - Ok(ServedRead::Mismatch(served)) => { + ServedRead::Mismatch(served) => { tracing::warn!( repo = %repo.name, requested = %ctx.canonical_cid, served = %served, "withholding object: served bytes do not hash to the requested CID (legacy provider-CID row?)" ); return GateOutcome::Skip; } - Ok(ServedRead::Gone) => return GateOutcome::Skip, - Ok(ServedRead::ReadErr(e)) => { + ServedRead::Gone => return GateOutcome::Skip, + ServedRead::ReadErr(e) => { // Infra failure (git spawn/IO), NOT a not-found: mark the search truncated so // a wholly-unserved request tails to a retryable 503, never a definitive 404 // for an authorized caller (INV-25 spirit — logging alone is not surfacing). @@ -718,11 +754,6 @@ async fn gate_and_serve( walk.truncated = true; return GateOutcome::Skip; } - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "object read task panicked"); - walk.truncated = true; - return GateOutcome::Skip; - } }; let mut resp_headers = HeaderMap::new(); resp_headers.insert( diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 31843bbc..4d6b9704 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -96,6 +96,22 @@ pub async fn pin_new_objects( for sha in object_list { match db.has_pinata_cid(&sha).await { Ok(true) => { + // Backfill NULL first-pinner provenance from a known source, in lockstep + // with the ipfs_pin skip branch: a pinata-only node otherwise leaves + // pre-provenance rows' `pinned_cids.repo_id` NULL forever (grok P2-D). The + // resolver still finds the object via the pin_repo_sources union below, so + // this is a consistency backfill, not a correctness fix. + match db.provenance_for_oid(&sha).await { + Ok(None) => { + if let Err(e) = db.backfill_pin_provenance(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to backfill pin provenance"); + } + } + Ok(Some(_)) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "DB error reading pin provenance"); + } + } // F1 (#173 round 8): record this repo as an additional source for the // already-pinned object (mirrors the ipfs_pin skip-branch insert) so the // resolver can serve a shared object from any pin-path source. From bb3ecc8f309746c3cd2995d87144fcbf97d284be Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 17 Jul 2026 09:27:56 -0500 Subject: [PATCH 04/77] fix(node): tie the /ipfs walk ceiling to MAX_PIN_SOURCES + 1 A provenanced object has a bounded source set (first-pinner + up to MAX_PIN_SOURCES additional). With the walk ceiling at 16 == MAX_PIN_SOURCES, an authorizing public source sorting after 16 path-scoped denials was never reached: the ceiling truncated the search and returned a false 503 for a readable object. Raise the ceiling to MAX_PIN_SOURCES + 1 so the whole bounded provenance set is always tried; the legacy scan stays bounded by MAX_LEGACY_PROBES_PER_REQUEST. --- crates/gitlawb-node/src/api/ipfs.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index cb18d52a..9cb663b2 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -50,7 +50,15 @@ use crate::visibility::{visibility_check, Decision}; /// serves on the first repo that grants them, so reaching it requires being /// denied by this many path-scoped repos first, which real traffic effectively /// never does. Tunable if that assumption stops holding. -pub(crate) const MAX_HISTORY_WALKS_PER_REQUEST: u32 = 16; +/// +/// Kept at `MAX_PIN_SOURCES + 1` so the ceiling can never truncate a request +/// BEFORE its whole bounded provenance source set (first-pinner + up to +/// `MAX_PIN_SOURCES` additional) has been tried: an authorizing public source that +/// sorts after `MAX_PIN_SOURCES` path-scoped denials must still be reached and +/// served, not falsely 503'd as a truncated search. The legacy scan's fan-out is +/// separately bounded by `MAX_LEGACY_PROBES_PER_REQUEST`, so widening this by one +/// does not loosen that path. +pub(crate) const MAX_HISTORY_WALKS_PER_REQUEST: u32 = crate::db::MAX_PIN_SOURCES as u32 + 1; /// Hard per-request ceiling on how many legacy (NULL-provenance) repositories /// the CID resolver's scan fallback may PROBE (`acquire` + `git cat-file -t`). From 107abf9145ab072331e492964ccf59e4a63baf3b Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 17 Jul 2026 09:37:43 -0500 Subject: [PATCH 05/77] fix(node): align ipfs_pin return with pinata; add Retry-After to SearchIncomplete - ipfs_pin::pin_new_objects returns the provider Hash (not the raw resolver key), matching the pinata twin's contract; the DB cid stays the raw content CID. The return is logging-only here, so this is drift-avoidance, not a functional fix. - AppError::SearchIncomplete now carries Retry-After: 1 like Overloaded, so both retryable 503s from the /ipfs handler advertise the retry hint consistently. --- crates/gitlawb-node/src/error.rs | 12 ++++++++---- crates/gitlawb-node/src/ipfs_pin.rs | 6 +++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index 371f0cf6..c13b92b4 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -182,10 +182,14 @@ impl IntoResponse for AppError { })); let mut resp = (status, body).into_response(); - // Overloaded advertises when to retry. It rides the shared tail above for - // its body/status, so the header is attached here rather than in a bespoke - // early return — keeping the variant handled in exactly one place. - if matches!(self, AppError::Overloaded(_)) { + // Both retryable 503s advertise when to retry: Overloaded (capacity shed) and + // SearchIncomplete (a bounded CID search cut short by a cap — retry may complete + // it). They ride the shared tail above for body/status, so the header is attached + // here rather than in bespoke early returns, keeping each variant handled once. + if matches!( + self, + AppError::Overloaded(_) | AppError::SearchIncomplete(_) + ) { resp.headers_mut().insert( axum::http::header::RETRY_AFTER, axum::http::HeaderValue::from_static("1"), diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index bdc6e090..e70d8f7b 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -175,7 +175,11 @@ pub async fn pin_new_objects( if let Err(e) = db.record_pin_source(&sha, repo_id).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); } - pinned.push((sha, raw_cid)); + // Return the provider Hash (not the resolver key), mirroring the pinata + // twin's contract: the DB `cid` is the raw resolver key (recorded above), + // the returned value is the provider CID. Here the return is consumed only + // for logging, but keeping the twins structurally identical avoids drift. + pinned.push((sha, cid)); } Ok(_) => {} Err(e) => { From 1e2f0d1dd6d090c1cb0df79d18f9b0a50bd4c4be Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:04:04 -0500 Subject: [PATCH 06/77] fix(node): close the /ipfs pin-source griefing hole via a bounded scan fallback An attacker who pins a public object from MAX_PIN_SOURCES repos before the legitimate public source registers fills pin_repo_sources (record_pin_source is first-N-wins and drops later sources silently); the buried public source is then never reached because a non-empty provenance set suppressed the legacy scan, so anon GET /ipfs/{cid} 404s forever for a public object (re-breaks F1). U1: db::pin_sources_at_cap reports whether pin_repo_sources is at MAX_PIN_SOURCES for an oid (the only observable signal that a servable source may have been dropped, since the write cap never overshoots). U2: get_by_cid falls back to the bounded legacy scan on a provenance miss when the set is empty OR at_cap. The scan gates every repo through the real per-caller gate, so it finds the buried public copy. A complete (non-full) set still fast-404s, so ordinary denials never fan out to O(repos) (INV-10/F3); the fallback honors the is_throttled peek. Regression ipfs_cid_buried_public_source_still_serves_via_scan_fallback: 404 before, 200 after; pin_sources_at_cap_flips_at_max covers the boundary. --- crates/gitlawb-node/src/api/ipfs.rs | 127 +++++++++++++----------- crates/gitlawb-node/src/db/mod.rs | 19 ++++ crates/gitlawb-node/src/test_support.rs | 123 +++++++++++++++++++++++ 3 files changed, 213 insertions(+), 56 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 9cb663b2..c8911bac 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -246,15 +246,77 @@ pub async fn get_by_cid( .pin_sources_for_oid(sha256_hex) .await .map_err(AppError::Internal)?; - if sources.is_empty() { - // F3 (#173, INV-10/INV-15): the legacy NULL-provenance scan builds an - // O(repos) preload (repos + rules + quarantine) BEFORE gate_and_serve's - // per-probe brake can bite, so a throttled source could still force O(repos) - // DB work on every replay. Peek the per-IP limiter WITHOUT consuming a token: - // an already-throttled source is shed here, before the preload runs. The + // Provenance fast-path: try each recorded source repo through the SAME gate + // (bounded to first-pinner + MAX_PIN_SOURCES). Empty for a legacy NULL-provenance + // pin. The first source that authorizes serves — no scan fan-out on the common + // path. + for repo_id in &sources { + let repo = match state + .db + .get_repo_by_id(repo_id) + .await + .map_err(AppError::Internal)? + { + Some(r) => r, + // A source repo is gone: skip it; a later source or the scan fallback + // below may still resolve. + None => continue, + }; + let quarantined = state + .db + .is_repo_quarantined(repo_id) + .await + .map_err(AppError::Internal)?; + let rules_map = state + .db + .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) + .await + .map_err(AppError::Internal)?; + let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); + match gate_and_serve( + &state, + &repo, + rules, + quarantined, + sha256_hex, + &rctx, + &mut walk, + false, + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + GateOutcome::Throttled => { + throttled = true; + continue; + } + GateOutcome::Skip => continue, + } + } + + // Bounded legacy-scan fallback. Run it when the provenance set could not have + // served the caller AND may be INCOMPLETE: + // - empty -> a legacy NULL-provenance pin (recorded before provenance existed), or + // - at_cap -> `record_pin_source` stops inserting at MAX_PIN_SOURCES and drops + // later sources SILENTLY, so a full table may hide a servable source + // (e.g. a later PUBLIC pinner buried by 16 attacker sources — the + // pin-source griefing hole). The scan gates every repo through the + // real per-caller gate, so it finds that copy. + // A non-empty, non-full set is COMPLETE (every recorded source was just tried), so + // skip the scan and let the tail 404 — ordinary denials never fan out to O(repos) + // (INV-10 / F3). The at_cap query runs only on a provenance MISS (we return above + // on Served), so it never costs the serve path. + let needs_scan = sources.is_empty() + || state + .db + .pin_sources_at_cap(sha256_hex) + .await + .map_err(AppError::Internal)?; + if needs_scan { + // F3 (#173, INV-10/INV-15): peek the per-IP limiter WITHOUT consuming a token + // so an already-throttled source is shed BEFORE the O(repos) preload; the // consuming per-probe charge inside gate_and_serve is left UNCHANGED (it is - // load-bearing for the across-request bound), so this adds no double-charge — - // a non-consuming peek plus the existing per-probe charge, never two charges. + // load-bearing for the across-request bound), so this adds no double-charge. if let Some(key) = crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) { @@ -263,9 +325,7 @@ pub async fn get_by_cid( continue; } } - // Legacy pin (recorded before provenance existed): fall back to the repo - // scan, gating each repo through the SAME gate. Load the scan context - // once, lazily. + // Load the scan context once, lazily (shared across oid candidates). if scan_ctx.is_none() { #[cfg(test)] bump_preload_queries(); @@ -309,51 +369,6 @@ pub async fn get_by_cid( GateOutcome::Skip => {} } } - } else { - for repo_id in &sources { - let repo = match state - .db - .get_repo_by_id(repo_id) - .await - .map_err(AppError::Internal)? - { - Some(r) => r, - // A source repo is gone: skip this source. Do NOT fall back to the - // scan (that would reopen the fan-out); a later source or oid - // candidate may still resolve. - None => continue, - }; - let quarantined = state - .db - .is_repo_quarantined(repo_id) - .await - .map_err(AppError::Internal)?; - let rules_map = state - .db - .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) - .await - .map_err(AppError::Internal)?; - let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); - match gate_and_serve( - &state, - &repo, - rules, - quarantined, - sha256_hex, - &rctx, - &mut walk, - false, // provenance path: bounded source set, no scan fan-out - ) - .await - { - GateOutcome::Served(resp) => return Ok(resp), - GateOutcome::Throttled => { - throttled = true; - continue; - } - GateOutcome::Skip => continue, - } - } } } diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 6b0d394e..8d723ffd 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -2379,6 +2379,25 @@ impl Db { .collect()) } + /// Whether `pin_repo_sources` is at the `MAX_PIN_SOURCES` cap for this oid, i.e. + /// the provenance source set returned by [`Self::pin_sources_for_oid`] may be + /// INCOMPLETE. `record_pin_source` stops inserting at exactly `MAX_PIN_SOURCES` + /// rows and drops later sources silently, so a full table is the only observable + /// signal that a servable source (e.g. a later public pinner) may have been + /// dropped. `get_by_cid` uses this to decide whether a provenance miss should fall + /// back to the bounded legacy scan (which gates every repo through the real + /// visibility gate and so finds a dropped public source) rather than 404 — closing + /// the pin-source griefing hole where 16 attacker sources bury a public one. `>=` + /// (not `==`) is defensive against any future overshoot. + pub async fn pin_sources_at_cap(&self, sha256_hex: &str) -> Result { + let count: i64 = + sqlx::query_scalar("SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_one(&self.pool) + .await?; + Ok(count >= MAX_PIN_SOURCES) + } + pub async fn record_encrypted_blob( &self, repo_id: &str, diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 21fa58e4..45b2b807 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -2450,6 +2450,129 @@ mod tests { ); } + /// U1 (grok round-4 P1): `pin_sources_at_cap` flips exactly at `MAX_PIN_SOURCES`. + /// It is the signal `get_by_cid` uses to decide a provenance miss may be hiding a + /// dropped servable source and must fall back to the bounded scan. + #[sqlx::test] + async fn pin_sources_at_cap_flips_at_max(pool: PgPool) { + let state = test_state(pool).await; + let cap = crate::db::MAX_PIN_SOURCES; + assert!( + !state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "an oid with no pin_repo_sources rows is not at cap" + ); + for i in 0..(cap - 1) { + state + .db + .record_pin_source("atcapoid", &format!("r-{i:02}")) + .await + .unwrap(); + } + assert!( + !state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "one below MAX_PIN_SOURCES is not at cap" + ); + state + .db + .record_pin_source("atcapoid", "r-last") + .await + .unwrap(); + assert!( + state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "exactly MAX_PIN_SOURCES rows is at cap" + ); + } + + /// U2 (grok round-4 P1, load-bearing): the pin-source GRIEFING hole. A private + /// first-pinner denies anon; an attacker fills the whole `MAX_PIN_SOURCES` source + /// window with deny-anon sources BEFORE a legitimate public repo pins the same + /// object, so the public repo's `record_pin_source` no-ops (cap full) and it is + /// buried — present in NO provenance record. The resolver's provenance set is then + /// {private + 16 attacker}, all deny anon. Because the set is at_cap (may hide a + /// dropped source), the handler falls back to the bounded legacy scan, which gates + /// every repo through the real gate and finds the buried PUBLIC copy → 200. + /// MUTATION (RED): remove the `at_cap` fallback edge in `get_by_cid` and the buried + /// public object 404s forever. + #[sqlx::test] + async fn ipfs_cid_buried_public_source_still_serves_via_scan_fallback(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["privfirst", "pubburied"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privfirst.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubburied.git"); + + // Private repo pins FIRST — owns the first-pinner provenance, denies anon. + let mut priv_repo = seed_repo(&owner_did, "privfirst"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private first-pinner"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // Attacker fills the ENTIRE MAX_PIN_SOURCES window with deny-anon (non-existent) + // sources BEFORE the public repo registers, so the cap is full. + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..cap { + state + .db + .record_pin_source(&fx.public_oid, &format!("00-attacker-{i:02}")) + .await + .expect("attacker source"); + } + + // A PUBLIC repo pushes the SAME object through the real pin path. Already pinned + // (skip branch), so it only tries record_pin_source — which NO-OPS because the + // cap is full. The public repo is thus buried: not the first-pinner, not in + // pin_repo_sources. + let pub_repo = seed_repo(&owner_did, "pubburied"); // public, no rule + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public buried source"); + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &pub_bare, + vec![fx.public_oid.clone()], + &state.db, + &pub_repo.id, + ) + .await; + m.assert_async().await; // /add NOT called (already pinned) + + // The buried public object must STILL serve: the provenance set is at_cap and + // all-deny, so the handler falls back to the bounded scan, which finds pubburied. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a public source buried by a full attacker source window must still serve via the bounded scan fallback (F1)" + ); + assert!( + body.contains("public bytes"), + "the served body is the buried public object's bytes" + ); + } + /// #173 (jatmn round 8, F1 — bound, R2): the per-object source set is capped at /// `MAX_PIN_SOURCES` so an adversary pushing one object from many repos cannot make /// resolution O(repos). Recording the same oid from `MAX_PIN_SOURCES + 3` distinct From de2ad5a83d372b1c24e8137306e4ad33ca5fc657 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:54:49 -0500 Subject: [PATCH 07/77] fix(node): hold read admission through the filtered-pack reaper The path-scoped upload-pack branch kept the read and per-caller permits as handler locals and called upload_pack_excluding with no AdmissionGuard, so a cancelled filtered clone released admission while KillGroupOnDrop was still reaping the git group, bypassing the read and per-caller concurrency caps. drive_git_child now returns the disarmed guard on success so one guard rides both build_filtered_pack stages (rev-list then pack-objects); the handler builds the guard from the permits as the plain path does. Adds a disconnect regression and an INV-22 gate row, both proven load-bearing. --- crates/gitlawb-node/src/api/repos.rs | 14 +- crates/gitlawb-node/src/git/smart_http.rs | 238 ++++++++++++++++++---- crates/gitlawb-node/tests/inv22_gates.rs | 16 ++ 3 files changed, 229 insertions(+), 39 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index cb6ed541..fb902f01 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -940,11 +940,15 @@ pub async fn git_upload_pack( smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); - // upload_pack_excluding runs its own rev-list/pack-objects (both pass `None` - // admission internally); the walk's permits stay handler-locals held across - // this serve, as be0cdd6 established, and drop when the handler returns. - let _hold = (_permit, _caller_permit); - smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout).await + // Move both admission permits into the guard so they release only after the + // filtered serve's git group (rev-list then pack-objects) is reaped, on + // complete/timeout/disconnect — not the instant a disconnect drops this + // future. Without this, disconnect-spam on a path-scoped repo could hold PIDs + // past the concurrency cap while the permits were already freed (#174 P1-a, + // R2). The guard rides both stages inside upload_pack_excluding. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout, Some(admission)) + .await } } .map_err(|e| { diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 64d3cb7f..ec115f1e 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -70,8 +70,9 @@ pub async fn info_refs( .arg("--stateless-rpc") .arg("--advertise-refs") .arg(repo_path); - // No request body — advertise-refs does not read stdin. - let stdout = + // No request body — advertise-refs does not read stdin. Single stage: the returned + // guard is dropped here (advertise-refs holds admission for its own child only). + let (stdout, _admission) = drive_git_child(command, Bytes::new(), timeout, "advertise-refs", admission).await?; let content_type = format!("application/x-{service}-advertisement"); @@ -331,23 +332,34 @@ async fn run_git_service( .arg(service_to_command(service)) .arg("--stateless-rpc") .arg(repo_path); - drive_git_child(command, input, timeout, service, admission).await + // Single stage: drop the returned guard when this function returns (its permits are + // released once the child's group is reaped, per drive_git_child). + let (out, _admission) = drive_git_child(command, input, timeout, service, admission).await?; + Ok(out) } /// Drive a spawned git child under `timeout` with process-group teardown, returning -/// its stdout. Shared core for [`run_git_service`] and [`info_refs`]: the caller -/// passes a `Command` with its args set; this adds piped stdio and `process_group(0)`. -/// On the deadline the whole group is torn down and reaped before returning -/// [`GitServiceTimeout`]; on a dropped future (client disconnect) the -/// [`KillGroupOnDrop`] guard fires. `input` is written to the child's stdin (empty -/// for the advertise-refs path, which has no request body); `what` labels errors. +/// its stdout AND the disarmed admission guard on success. Shared core for +/// [`run_git_service`] and [`info_refs`]: the caller passes a `Command` with its args +/// set; this adds piped stdio and `process_group(0)`. On the deadline the whole group +/// is torn down and reaped before returning [`GitServiceTimeout`]; on a dropped future +/// (client disconnect) the [`KillGroupOnDrop`] guard fires. `input` is written to the +/// child's stdin (empty for the advertise-refs path, which has no request body); `what` +/// labels errors. +/// +/// On success the guard is RETURNED rather than dropped internally (#174 KTD3), so a +/// caller running two sequential git stages under one admission — `build_filtered_pack`'s +/// rev-list then pack-objects — can hand the same guard from the first stage to the +/// second and keep the permits held across both, releasing them only when the second +/// stage's process group is reaped. Callers that run a single stage just let the +/// returned guard drop. async fn drive_git_child( mut command: Command, input: Bytes, timeout: Duration, what: &str, admission: Option, -) -> Result> { +) -> Result<(Vec, Option)> { command .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -383,9 +395,9 @@ async fn drive_git_child( admission, }; // On non-unix there is no process-group teardown, so hold the admission guard here - // for the child's whole interaction; it drops when this function returns. + // for the child's whole interaction; it is returned to the caller below. #[cfg(not(unix))] - let _admission = admission; + let admission_holder = admission; let mut out = Vec::new(); let mut err = Vec::new(); @@ -419,15 +431,24 @@ async fn drive_git_child( }; let timed = tokio::time::timeout(timeout, interact).await; + // The disarmed admission guard, handed back to the caller on success so it can ride + // a following sequential git stage (#174 KTD3) instead of releasing between stages. + let admission_out: Option; let (write_result, status) = match timed { Ok(result) => { // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the // guard's drop would reap an already-reaped child / signal a reused pgid. - // Dropping the returned admission guard here releases the permits at the - // earliest provably-free point on the success path (the op finished). + // The guard is returned (not dropped) so a two-stage caller keeps the + // permits held across both stages; a single-stage caller drops it on return. #[cfg(unix)] - drop(group_guard.disarm()); + { + admission_out = group_guard.disarm(); + } + #[cfg(not(unix))] + { + admission_out = admission_holder; + } result? } Err(_elapsed) => { @@ -460,7 +481,7 @@ async fn drive_git_child( write_result.context("failed to write to git stdin")?; - Ok(out) + Ok((out, admission_out)) } fn service_to_command(service: &str) -> &str { @@ -498,14 +519,18 @@ async fn rev_list_keep( repo_path: &Path, withheld: &HashSet, timeout: Duration, -) -> Result> { + admission: Option, +) -> Result<(Vec, Option)> { let mut command = Command::new(git_bin); command .args(["rev-list", "--objects", "--all"]) .current_dir(repo_path); - // The visibility-walk callers intentionally hold no admission permit (their - // admission is governed elsewhere), so pass `None` (#174 KTD2). - let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list", None).await?; + // First of `build_filtered_pack`'s two sequential stages: it holds the caller's + // admission for the rev-list child and hands the disarmed guard back so the same + // permits ride the pack-objects stage (#174 KTD3, R2). The inter-stage window holds + // no live git group, so nothing is reaped-late by carrying the guard between them. + let (stdout, admission) = + drive_git_child(command, Bytes::new(), timeout, "rev-list", admission).await?; let mut keep = Vec::new(); for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); @@ -514,7 +539,7 @@ async fn rev_list_keep( } keep.push(oid.to_string()); } - Ok(keep) + Ok((keep, admission)) } /// Build a packfile containing every object reachable from all refs EXCEPT the @@ -532,15 +557,21 @@ pub async fn build_filtered_pack( repo_path: &Path, withheld: &HashSet, timeout: Duration, + admission: Option, ) -> Result> { // One deadline spans both git stages so a slow rev-list eats into the pack // budget rather than granting each stage a fresh `timeout` (2x the permit hold). let deadline = Instant::now() + timeout; - let keep = rev_list_keep( + // The caller's admission rides both stages: rev-list holds it, hands it back, then + // pack-objects holds it until ITS process group is reaped on disconnect. That closes + // the path-scoped cap bypass — permits release on group reap, not the instant the + // request future drops (#174 R2, KTD3). The plain upload-pack path already does this. + let (keep, admission) = rev_list_keep( git_bin, repo_path, withheld, deadline.saturating_duration_since(Instant::now()), + admission, ) .await?; let mut data = keep.join("\n").into_bytes(); @@ -549,15 +580,15 @@ pub async fn build_filtered_pack( command .args(["pack-objects", "--stdout"]) .current_dir(repo_path); - drive_git_child( + let (out, _admission) = drive_git_child( command, Bytes::from(data), deadline.saturating_duration_since(Instant::now()), "pack-objects", - // Visibility-walk pack build: no admission permit here (#174 KTD2). - None, + admission, ) - .await + .await?; + Ok(out) } /// Serve a clone/fetch with the withheld blobs removed from the response pack. @@ -588,12 +619,15 @@ pub async fn upload_pack_excluding( request_body: Bytes, withheld: &HashSet, timeout: Duration, + admission: Option, ) -> Result { - // The rev-list enumeration runs blocking off the runtime; the streaming - // pack-objects stage is duration-bounded and its process group is reaped on + // Both git stages are duration-bounded and their process groups are reaped on // disconnect via drive_git_child (#174), so a hung build no longer pins its - // concurrency slot and a client disconnect no longer orphans the git child. - let pack = build_filtered_pack("git", repo_path, withheld, timeout).await?; + // concurrency slot and a client disconnect no longer orphans the git child. The + // caller's read + per-caller admission is threaded through both stages so the + // permits are held until the pack-objects group is reaped, matching the plain path + // (#174 R2). + let pack = build_filtered_pack("git", repo_path, withheld, timeout, admission).await?; // The client lists its capabilities on the first `want` line. Honor // side-band-64k when offered (every modern smart-HTTP client offers it); @@ -712,7 +746,7 @@ mod tests { let mut withheld = std::collections::HashSet::new(); withheld.insert(secret.clone()); - let pack = build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30)) + let pack = build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30), None) .await .unwrap(); let ids = pack_object_ids(&pack); @@ -776,7 +810,7 @@ mod tests { b"0098want 0000000000000000000000000000000000000000 \ side-band-64k ofs-delta agent=git/2\n00000009done\n", ); - let resp = upload_pack_excluding(&bare, req, &withheld, Duration::from_secs(30)) + let resp = upload_pack_excluding(&bare, req, &withheld, Duration::from_secs(30), None) .await .unwrap(); let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); @@ -837,7 +871,7 @@ mod tests { axum::extract::State(st): axum::extract::State>, body: Bytes, ) -> Response { - upload_pack_excluding(&st.repo, body, &st.withheld, Duration::from_secs(30)) + upload_pack_excluding(&st.repo, body, &st.withheld, Duration::from_secs(30), None) .await .unwrap() } @@ -1359,7 +1393,7 @@ mod tests { for i in 0..FAKE_GIT_RETRY_ATTEMPTS { let result = tokio::time::timeout( Duration::from_secs(10), - build_filtered_pack(git_bin, repo_path, withheld, stage_timeout), + build_filtered_pack(git_bin, repo_path, withheld, stage_timeout, None), ) .await .expect( @@ -1655,6 +1689,142 @@ mod tests { ); } + // #174 U1 (R2, KTD3, RED-before/GREEN-after): the path-scoped filtered-pack serve + // must hold read + per-caller admission until its pack-objects process group is + // reaped on a client disconnect, exactly as the plain upload_pack path does. Before + // the fix build_filtered_pack took no AdmissionGuard and the handler's `_hold` + // permits dropped the instant the request future was dropped, so disconnect-spam on + // a path-scoped repo could hold PIDs past the concurrency cap while the permits were + // already free (#174 P1-a, on the filtered path the plain path had already closed). + // + // A real AdmissionGuard built from two owned semaphore permits rides + // rev-list -> pack-objects. We drive the future until pack-objects has forked its + // grandchild (the streaming pack-writer stand-in), assert the permits are still held + // mid-serve, then DROP the future (client disconnect) and assert the permits are + // released only AFTER the group is ESRCH-confirmed gone — never while it is alive. + // Goes RED if the guard is not threaded into the pack-objects stage (it would then + // drop after rev-list, freeing the permits mid-serve or on the bare future drop). + #[cfg(unix)] + #[tokio::test] + async fn filtered_pack_holds_admission_until_group_reaped_on_disconnect() { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("pids"); + // rev-list returns one oid fast; pack-objects forks a grandchild (the streaming + // writer stand-in), records leader+grandchild pids, then hangs so the future + // parks mid-serve with the guard owned by the pack-objects KillGroupOnDrop. The + // grandchild inherits (holds open) the stdout pipe, so drive_git_child's + // read_to_end blocks and the future stays pending until we drop it. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 &\nprintf '%s\\n%s\\n' \"$$\" \"$!\" > \"{}\"\nwait ;;\n *) exit 1 ;;\nesac\n", + pidfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let withheld = HashSet::new(); + + // Retry the fake-git spawn race like the sibling disconnect test; each attempt + // gets a FRESH semaphore so a dropped losing attempt can't skew the winning + // attempt's permit accounting. Keep the winning attempt's future PENDING so the + // drop below exercises the client-disconnect teardown. + let (fut, sem, leader, grandchild) = { + let mut attempt = 0u64; + loop { + attempt += 1; + let _ = std::fs::remove_file(&pidfile); + // Semaphore(4): two owned permits model the handler's global-read + + // per-caller admission, leaving 2 available while the op is in flight. + let sem = Arc::new(Semaphore::new(4)); + let g = sem.clone().try_acquire_owned().unwrap(); + let c = sem.clone().try_acquire_owned().unwrap(); + let admission = AdmissionGuard::new(g, Some(c)); + let mut fut = Box::pin(build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(60), + Some(admission), + )); + // Advance the future a slice at a time until the fake records its pids + // (i.e. pack-objects is running). `Ok(_)` means the future returned + // before the pidfile appeared (spawn error / early exit); stop polling + // then, since re-polling a completed future panics. + let mut pids = None; + for _ in 0..500 { + let finished = + tokio::time::timeout(Duration::from_millis(10), &mut fut).await.is_ok(); + if let Some(p) = read_two_pids(&pidfile) { + pids = Some(p); + break; + } + if finished { + break; + } + } + match pids { + Some((l, gch)) => break (fut, sem, l, gch), + None => { + // Transient spawn miss: drop the still-armed future so its guard + // reaps anything that spawned (and returns the permits), then + // back off before retrying. + drop(fut); + assert!( + attempt < FAKE_GIT_RETRY_ATTEMPTS, + "fake git failed to reach the pack-objects stage after \ + {FAKE_GIT_RETRY_ATTEMPTS} attempts (persistent failure, \ + not a transient parallel-runner miss)" + ); + tokio::time::sleep(Duration::from_millis( + FAKE_GIT_BACKOFF_STEP_MS * attempt, + )) + .await; + } + } + } + }; + let _cleanup = ReapOnPanic(vec![leader, grandchild]); + assert!(alive(grandchild), "grandchild must be running mid-serve"); + + // Mid-serve: the pack-objects stage owns the guard, so the two permits are still + // held. A build that dropped the guard after rev-list (unthreaded pack-objects + // stage) would have freed them here. + assert_eq!( + sem.available_permits(), + 2, + "admission permits must be held while the filtered serve is in flight" + ); + + // Client disconnect: drop the request future. The pack-objects KillGroupOnDrop + // must tear the group down AND hold the permits until the group is + // ESRCH-confirmed gone, releasing them only then. + drop(fut); + + let mut released_while_alive = false; + let mut released_after_reap = false; + for _ in 0..500 { + let released = sem.available_permits() == 4; + let group_alive = alive(grandchild); + if released && group_alive { + released_while_alive = true; + } + if released && !group_alive { + released_after_reap = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!( + !released_while_alive, + "admission permits were released while the git group was still alive — the \ + path-scoped concurrency-cap bypass (#174 P1-a) is open" + ); + assert!( + released_after_reap, + "admission permits must be released once the group is reaped on disconnect" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 7d3fd1c8..e750a48f 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -68,6 +68,22 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { withheld_recipients_gated, which acquires git_encrypt_semaphore" ); + // U1 / R2 (#173 round-10): the path-scoped filtered-pack serve must thread the + // caller's AdmissionGuard through BOTH git stages so read + per-caller admission is + // held until the pack-objects group is reaped on disconnect, closing the cap bypass + // the plain upload_pack path already fixed. Two load-bearing markers: rev-list hands + // the disarmed guard back (its tuple return type), and build_filtered_pack forwards + // that guard into the pack-objects stage (the `admission` arg after the + // "pack-objects" label). Reverting either — dropping the guard between stages, or + // passing `None` to pack-objects — trips this. + assert!( + smart_http.contains("Result<(Vec, Option)>") + && smart_http.contains("\"pack-objects\",\n admission,"), + "U1/R2 gate missing: build_filtered_pack must thread the AdmissionGuard through \ + rev-list -> pack-objects so the permits are held until the pack-objects group \ + is reaped on disconnect (the path-scoped half of #174 P1-a)" + ); + // P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere // but inside withheld_recipients_gated. A second call site (count > 1) is a new // detached git walk that skips the admission gate — exactly the class U5 closed. From b881447b598127e4b16318da55fc0be8accfa42e Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:34:09 -0500 Subject: [PATCH 08/77] fix(node): own /ipfs admission for the life of the walk get_by_cid held the global walk permit and the per-source permit as handler locals, so dropping the request future released both while the spawn_blocking probe, visibility walk, and object read kept running; a cancel-spam or timeout client could exceed MAX_CONCURRENT_IPFS_WALKS and the per-source cap, and the bare cat-file probe/read children had no teardown at all. Move the gated serve pipeline into a detached task that owns an AdmissionGuard, awaited by the handler, so admission releases only after the bounded work is gone; back the /ipfs probe and read with run_bounded_git twins for process-group teardown. Adds cancel-mid-walk, timeout-reap, and cancel-spam regressions plus two load-bearing INV-22 rows. Also reflows one U1 test line to satisfy cargo fmt. --- crates/gitlawb-node/src/api/ipfs.rs | 749 +++++++++++++--------- crates/gitlawb-node/src/git/smart_http.rs | 5 +- crates/gitlawb-node/src/git/store.rs | 184 +++++- crates/gitlawb-node/tests/inv22_gates.rs | 33 + 4 files changed, 661 insertions(+), 310 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index c8911bac..016e130d 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -165,7 +165,7 @@ pub async fn get_by_cid( // admits any `did:key` unthrottled, so a DID key would be free to mint around); a // `None` key (no trusted header, no peer) is bounded by the global pool only, // never the per-source sub-cap. - let _ipfs_walk_permit = state + let ipfs_walk_permit = state .git_ipfs_walk_semaphore .clone() .try_acquire_owned() @@ -174,7 +174,7 @@ pub async fn get_by_cid( AppError::Overloaded("ipfs service at capacity, retry shortly".into()) })?; let source_key = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust); - let _ipfs_caller_permit = match &source_key { + let ipfs_caller_permit = match &source_key { Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); AppError::Overloaded("ipfs service at capacity for this source, retry shortly".into()) @@ -182,219 +182,253 @@ pub async fn get_by_cid( None => None, }; - // Resolve the content-addressed CID to the object's git oid(s). A real pin - // CID digests the raw object content (`Cid::from_git_object_bytes`), NOT the - // git oid (git frames content with a `" \0"` header first), so we - // map it back through `pinned_cids` rather than treating the digest as an oid - // (#173). The cid index is non-unique, so one CID can map to several oids (a - // tree and a blob whose raw bytes collide, or content pinned under two oids); - // we try each candidate below rather than pick one arbitrarily and false-404 - // when the chosen one is withheld or absent while another is readable (#173). - // An empty result is an opaque 404, uniform with a genuine not-found and a - // visibility denial. - let oids = state - .db - .oids_for_cid(&canonical_cid) - .await - .map_err(AppError::Internal)?; - if oids.is_empty() { - return Err(AppError::RepoNotFound(format!( - "no git object found for CID {cid_str}" - ))); - } - let caller = auth.as_ref().map(|e| e.0 .0.as_str()); - let caller_owned = caller.map(|c| c.to_string()); - - // Per-request walk budget + memos + throttle flag, shared by the provenance path - // and the legacy scan so both honor the same fan-out ceiling, per-repo memo, and - // IP brake. The caller is constant for one request, so `repo.id` alone keys the memo. - let mut walk = WalkState { - walks: 0, - probes: 0, - truncated: false, - allowed_blob_memo: HashMap::new(), - allowed_tree_memo: HashMap::new(), - reachable_ct_memo: HashMap::new(), - }; - // Set when a walk-requiring candidate is skipped because the source IP's walk quota - // is spent (#173 review, F-C): the scan keeps going so a later walk-free copy still - // serves; only if nothing is servable is it turned into the 429. - let mut throttled = false; - let rctx = ResolveCtx { - caller, - caller_owned: &caller_owned, - headers: &headers, - peer, - cid_str: &cid_str, - canonical_cid: &canonical_cid, - }; + // Caller DID (owned) resolved before the spawn: the detached task below cannot + // borrow the handler's `auth` extension. + let caller_owned = auth.as_ref().map(|e| e.0 .0.as_str().to_string()); + + // #173 round-10 (R1, KTD1): move BOTH admission permits into an AdmissionGuard OWNED + // by a detached tokio task that runs the whole gated serve pipeline, and have the + // handler await its JoinHandle. Dropping the request future on a client disconnect + // drops the JoinHandle, which DETACHES the task (tokio does not abort a task when its + // handle drops) rather than cancelling it — so the pipeline runs to its bounded + // completion and releases admission only then, instead of the permits dropping the + // instant the handler future is torn down while a spawn_blocking git probe/walk/read + // is still alive (the disconnect-spam cap bypass this closes, the /ipfs half of #174 + // P1-a). Every git child on this pipeline is duration-bounded (the run_bounded_git + // probe/read twins + the bounded walk), so the detached task cannot hold admission + // past ~git_service_timeout_secs. The client key is already captured (`source_key`) + // before the spawn; the detached task has no request extractors. + let admission = + crate::git::smart_http::AdmissionGuard::new(ipfs_walk_permit, ipfs_caller_permit); + let serve: tokio::task::JoinHandle> = tokio::spawn(async move { + // The guard is held for the whole task and drops LAST — after the response is + // built — releasing admission exactly once when the pipeline is truly done. + let _admission = admission; - // Legacy scan context (repos + rules + quarantined ids), loaded LAZILY only when a - // legacy NULL-provenance pin is hit — the provenance path must never trigger the - // O(repos) load (that fan-out is exactly what provenance removes, #173 round 2). - let mut scan_ctx: Option = None; - - for sha256_hex in &oids { - // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). - // Resolve a PROVENANCED pin by trying each source repo (bounded to - // MAX_PIN_SOURCES) through the SAME gate; the first that authorizes serves — no - // scan fan-out. A shared object first pinned from a private/quarantined repo - // still serves from a later PUBLIC source. Deterministic (ORDER BY on the - // union), so no ordering can turn an authorized copy into a 404. - let sources = state + // Resolve the content-addressed CID to the object's git oid(s). A real pin + // CID digests the raw object content (`Cid::from_git_object_bytes`), NOT the + // git oid (git frames content with a `" \0"` header first), so we + // map it back through `pinned_cids` rather than treating the digest as an oid + // (#173). The cid index is non-unique, so one CID can map to several oids (a + // tree and a blob whose raw bytes collide, or content pinned under two oids); + // we try each candidate below rather than pick one arbitrarily and false-404 + // when the chosen one is withheld or absent while another is readable (#173). + // An empty result is an opaque 404, uniform with a genuine not-found and a + // visibility denial. + let oids = state .db - .pin_sources_for_oid(sha256_hex) + .oids_for_cid(&canonical_cid) .await .map_err(AppError::Internal)?; - // Provenance fast-path: try each recorded source repo through the SAME gate - // (bounded to first-pinner + MAX_PIN_SOURCES). Empty for a legacy NULL-provenance - // pin. The first source that authorizes serves — no scan fan-out on the common - // path. - for repo_id in &sources { - let repo = match state - .db - .get_repo_by_id(repo_id) - .await - .map_err(AppError::Internal)? - { - Some(r) => r, - // A source repo is gone: skip it; a later source or the scan fallback - // below may still resolve. - None => continue, - }; - let quarantined = state - .db - .is_repo_quarantined(repo_id) - .await - .map_err(AppError::Internal)?; - let rules_map = state - .db - .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) - .await - .map_err(AppError::Internal)?; - let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); - match gate_and_serve( - &state, - &repo, - rules, - quarantined, - sha256_hex, - &rctx, - &mut walk, - false, - ) - .await - { - GateOutcome::Served(resp) => return Ok(resp), - GateOutcome::Throttled => { - throttled = true; - continue; - } - GateOutcome::Skip => continue, - } + if oids.is_empty() { + return Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))); } + let caller = caller_owned.as_deref(); + + // Per-request walk budget + memos + throttle flag, shared by the provenance path + // and the legacy scan so both honor the same fan-out ceiling, per-repo memo, and + // IP brake. The caller is constant for one request, so `repo.id` alone keys the memo. + let mut walk = WalkState { + walks: 0, + probes: 0, + truncated: false, + allowed_blob_memo: HashMap::new(), + allowed_tree_memo: HashMap::new(), + reachable_ct_memo: HashMap::new(), + }; + // Set when a walk-requiring candidate is skipped because the source IP's walk quota + // is spent (#173 review, F-C): the scan keeps going so a later walk-free copy still + // serves; only if nothing is servable is it turned into the 429. + let mut throttled = false; + let rctx = ResolveCtx { + caller, + caller_owned: &caller_owned, + headers: &headers, + peer, + cid_str: &cid_str, + canonical_cid: &canonical_cid, + }; + + // Legacy scan context (repos + rules + quarantined ids), loaded LAZILY only when a + // legacy NULL-provenance pin is hit — the provenance path must never trigger the + // O(repos) load (that fan-out is exactly what provenance removes, #173 round 2). + let mut scan_ctx: Option = None; - // Bounded legacy-scan fallback. Run it when the provenance set could not have - // served the caller AND may be INCOMPLETE: - // - empty -> a legacy NULL-provenance pin (recorded before provenance existed), or - // - at_cap -> `record_pin_source` stops inserting at MAX_PIN_SOURCES and drops - // later sources SILENTLY, so a full table may hide a servable source - // (e.g. a later PUBLIC pinner buried by 16 attacker sources — the - // pin-source griefing hole). The scan gates every repo through the - // real per-caller gate, so it finds that copy. - // A non-empty, non-full set is COMPLETE (every recorded source was just tried), so - // skip the scan and let the tail 404 — ordinary denials never fan out to O(repos) - // (INV-10 / F3). The at_cap query runs only on a provenance MISS (we return above - // on Served), so it never costs the serve path. - let needs_scan = sources.is_empty() - || state + for sha256_hex in &oids { + // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). + // Resolve a PROVENANCED pin by trying each source repo (bounded to + // MAX_PIN_SOURCES) through the SAME gate; the first that authorizes serves — no + // scan fan-out. A shared object first pinned from a private/quarantined repo + // still serves from a later PUBLIC source. Deterministic (ORDER BY on the + // union), so no ordering can turn an authorized copy into a 404. + let sources = state .db - .pin_sources_at_cap(sha256_hex) + .pin_sources_for_oid(sha256_hex) .await .map_err(AppError::Internal)?; - if needs_scan { - // F3 (#173, INV-10/INV-15): peek the per-IP limiter WITHOUT consuming a token - // so an already-throttled source is shed BEFORE the O(repos) preload; the - // consuming per-probe charge inside gate_and_serve is left UNCHANGED (it is - // load-bearing for the across-request bound), so this adds no double-charge. - if let Some(key) = - crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) - { - if state.ipfs_rate_limiter.is_throttled(&key).await { - throttled = true; - continue; - } - } - // Load the scan context once, lazily (shared across oid candidates). - if scan_ctx.is_none() { - #[cfg(test)] - bump_preload_queries(); - let repos = state + // Provenance fast-path: try each recorded source repo through the SAME gate + // (bounded to first-pinner + MAX_PIN_SOURCES). Empty for a legacy NULL-provenance + // pin. The first source that authorizes serves — no scan fan-out on the common + // path. + for repo_id in &sources { + let repo = match state .db - .list_all_repos() + .get_repo_by_id(repo_id) .await - .map_err(AppError::Internal)?; - let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); - let rules_by_repo = state + .map_err(AppError::Internal)? + { + Some(r) => r, + // A source repo is gone: skip it; a later source or the scan fallback + // below may still resolve. + None => continue, + }; + let quarantined = state .db - .list_visibility_rules_for_repos(&repo_ids) + .is_repo_quarantined(repo_id) .await .map_err(AppError::Internal)?; - let quarantined: HashSet = state + let rules_map = state .db - .list_quarantined_repos() + .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) .await - .map_err(AppError::Internal)? - .into_iter() - .map(|r| r.id) - .collect(); - scan_ctx = Some((repos, rules_by_repo, quarantined)); - } - let (repos, rules_by_repo, quarantined) = scan_ctx.as_ref().unwrap(); - for repo in repos { - let rules = rules_by_repo - .get(&repo.id) - .map(Vec::as_slice) - .unwrap_or(&[]); - let is_quar = quarantined.contains(&repo.id); + .map_err(AppError::Internal)?; + let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); match gate_and_serve( - &state, repo, rules, is_quar, sha256_hex, &rctx, &mut walk, true, + &state, + &repo, + rules, + quarantined, + sha256_hex, + &rctx, + &mut walk, + false, ) .await { GateOutcome::Served(resp) => return Ok(resp), - // A throttled walk-requiring candidate is skipped, not fatal: - // keep scanning for a later walk-free copy (#173 review, F-C). - GateOutcome::Throttled => throttled = true, - GateOutcome::Skip => {} + GateOutcome::Throttled => { + throttled = true; + continue; + } + GateOutcome::Skip => continue, + } + } + + // Bounded legacy-scan fallback. Run it when the provenance set could not have + // served the caller AND may be INCOMPLETE: + // - empty -> a legacy NULL-provenance pin (recorded before provenance existed), or + // - at_cap -> `record_pin_source` stops inserting at MAX_PIN_SOURCES and drops + // later sources SILENTLY, so a full table may hide a servable source + // (e.g. a later PUBLIC pinner buried by 16 attacker sources — the + // pin-source griefing hole). The scan gates every repo through the + // real per-caller gate, so it finds that copy. + // A non-empty, non-full set is COMPLETE (every recorded source was just tried), so + // skip the scan and let the tail 404 — ordinary denials never fan out to O(repos) + // (INV-10 / F3). The at_cap query runs only on a provenance MISS (we return above + // on Served), so it never costs the serve path. + let needs_scan = sources.is_empty() + || state + .db + .pin_sources_at_cap(sha256_hex) + .await + .map_err(AppError::Internal)?; + if needs_scan { + // F3 (#173, INV-10/INV-15): peek the per-IP limiter WITHOUT consuming a token + // so an already-throttled source is shed BEFORE the O(repos) preload; the + // consuming per-probe charge inside gate_and_serve is left UNCHANGED (it is + // load-bearing for the across-request bound), so this adds no double-charge. + if let Some(key) = + crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) + { + if state.ipfs_rate_limiter.is_throttled(&key).await { + throttled = true; + continue; + } + } + // Load the scan context once, lazily (shared across oid candidates). + if scan_ctx.is_none() { + #[cfg(test)] + bump_preload_queries(); + let repos = state + .db + .list_all_repos() + .await + .map_err(AppError::Internal)?; + let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); + let rules_by_repo = state + .db + .list_visibility_rules_for_repos(&repo_ids) + .await + .map_err(AppError::Internal)?; + let quarantined: HashSet = state + .db + .list_quarantined_repos() + .await + .map_err(AppError::Internal)? + .into_iter() + .map(|r| r.id) + .collect(); + scan_ctx = Some((repos, rules_by_repo, quarantined)); + } + let (repos, rules_by_repo, quarantined) = scan_ctx.as_ref().unwrap(); + for repo in repos { + let rules = rules_by_repo + .get(&repo.id) + .map(Vec::as_slice) + .unwrap_or(&[]); + let is_quar = quarantined.contains(&repo.id); + match gate_and_serve( + &state, repo, rules, is_quar, sha256_hex, &rctx, &mut walk, true, + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + // A throttled walk-requiring candidate is skipped, not fatal: + // keep scanning for a later walk-free copy (#173 review, F-C). + GateOutcome::Throttled => throttled = true, + GateOutcome::Skip => {} + } } } } - } - // Nothing served — three distinct tails, in precedence order: - // 1. The scan was cut short by a cap (legacy probe ceiling or walk ceiling), so - // the object was NOT proven absent/unreadable everywhere → 503, retryable, and - // explicitly NOT a definitive not-found (#173, F2). This outranks the throttle: - // an incomplete search must not masquerade as a clean rate-limit outcome, and - // it carries only the caller-supplied CID (no object/OID/metadata leak). - // 2. A walk-requiring candidate was skipped for a spent IP quota while the scan - // otherwise completed → 429 (the brake bit; a cheaper copy was sought first). - // 3. A full scan under the caps found nothing readable → opaque 404, uniform with - // a genuine not-found and a visibility denial. - if walk.truncated { - return Err(AppError::SearchIncomplete(format!( - "CID {cid_str} search incomplete — retry" - ))); - } - if throttled { - return Err(AppError::TooManyRequests( - "ipfs retrieval rate limit exceeded — try again later".into(), - )); + // Nothing served — three distinct tails, in precedence order: + // 1. The scan was cut short by a cap (legacy probe ceiling or walk ceiling), so + // the object was NOT proven absent/unreadable everywhere → 503, retryable, and + // explicitly NOT a definitive not-found (#173, F2). This outranks the throttle: + // an incomplete search must not masquerade as a clean rate-limit outcome, and + // it carries only the caller-supplied CID (no object/OID/metadata leak). + // 2. A walk-requiring candidate was skipped for a spent IP quota while the scan + // otherwise completed → 429 (the brake bit; a cheaper copy was sought first). + // 3. A full scan under the caps found nothing readable → opaque 404, uniform with + // a genuine not-found and a visibility denial. + if walk.truncated { + return Err(AppError::SearchIncomplete(format!( + "CID {cid_str} search incomplete — retry" + ))); + } + if throttled { + return Err(AppError::TooManyRequests( + "ipfs retrieval rate limit exceeded — try again later".into(), + )); + } + Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))) + }); + + // Await the detached serve task. Dropping THIS future (client disconnect) drops the + // JoinHandle and detaches the task, which keeps running and releases admission only + // when it completes (KTD1). A JoinError here is a panic in the pipeline (the task is + // never aborted), surfaced as a 500 rather than a silent hang. + match serve.await { + Ok(result) => result, + Err(join_err) => Err(AppError::Internal(anyhow::anyhow!( + "ipfs serve task failed: {join_err}" + ))), } - Err(AppError::RepoNotFound(format!( - "no git object found for CID {cid_str}" - ))) } /// Outcome of gating one repo for one candidate oid. @@ -553,32 +587,34 @@ async fn gate_and_serve( let obj_type = { let rp = repo_path.clone(); let sha = sha256_hex.to_string(); - // Bound the blocking `git cat-file -t` under `git_service_timeout_secs`: this probe - // runs while the /ipfs walk permit is held, so a wedged cat-file (corrupt pack, NFS - // stall) would otherwise pin the global walk slot for the request's life. On timeout - // free the slot (mark truncated -> retryable 503) and skip the repo. spawn_blocking - // cannot be cancelled, so the child may linger on a blocking-pool thread, but it no - // longer holds the walk permit. - let probe_deadline = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - match tokio::time::timeout( - probe_deadline, - tokio::task::spawn_blocking(move || store::object_type(&rp, &sha)), - ) + let git_bin = state.git_bin.clone(); + // Bound the probe CHILD itself (process-group teardown at `git_service_timeout_secs` + // via `object_type_bounded` -> `run_bounded_git`), not just an outer tokio timeout + // racing an uncancellable `spawn_blocking`: this probe runs while the /ipfs walk + // permit is held by the owning task, so a wedged cat-file (corrupt pack, NFS stall) + // must be REAPED at the deadline rather than left to linger and delay the task's + // completion — and thus admission release (#173 round-10, KTD2). No outer timeout, + // mirroring the bounded walk below; a `GitServiceTimeout` marks the search truncated + // (retryable 503). + let probe_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + match tokio::task::spawn_blocking(move || { + store::object_type_bounded(&git_bin, &rp, &sha, probe_timeout) + }) .await { - Ok(Ok(Ok(Some(t)))) => t, - Ok(Ok(Ok(None))) => return GateOutcome::Skip, - Ok(Ok(Err(e))) => { - tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); - return GateOutcome::Skip; - } + Ok(Ok(Some(t))) => t, + Ok(Ok(None)) => return GateOutcome::Skip, Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "object-type probe task panicked; skipping repo"); + if e.is::() { + tracing::warn!(repo = %repo.name, "object-type probe timed out under the /ipfs walk permit; skipping repo"); + walk.truncated = true; + } else { + tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); + } return GateOutcome::Skip; } - Err(_elapsed) => { - tracing::warn!(repo = %repo.name, "object-type probe timed out under the /ipfs walk permit; skipping repo"); - walk.truncated = true; + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "object-type probe task panicked; skipping repo"); return GateOutcome::Skip; } } @@ -710,45 +746,49 @@ async fn gate_and_serve( let read_sha = sha256_hex.to_string(); let read_type = obj_type.clone(); let want_cid = ctx.canonical_cid.to_string(); - // Bound the blocking size+read+verify under `git_service_timeout_secs` (same rationale - // as the object-type probe): a hung cat-file must not pin the held /ipfs walk permit. - let read_deadline = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let read = tokio::time::timeout( - read_deadline, - tokio::task::spawn_blocking(move || -> ServedRead { - match store::object_size(&read_repo, &read_sha) { - Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), - Ok(Some(_)) => {} - // git ran and reported no such object (or an unparseable size): genuine - // not-found for this candidate. - Ok(None) => return ServedRead::Gone, - // git itself failed to run: an infra failure, not a not-found. - Err(e) => return ServedRead::ReadErr(e.to_string()), - } - let content = match store::read_object_content(&read_repo, &read_sha, &read_type) { - Ok(c) => c, - Err(e) => return ServedRead::ReadErr(e.to_string()), - }; - let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); - if served != want_cid { - return ServedRead::Mismatch(served); - } - ServedRead::Ok(content) - }), - ) + let git_bin = state.git_bin.clone(); + // Bound the size+read CHILDREN themselves (process-group teardown at + // `git_service_timeout_secs` via the `*_bounded` twins), not an outer tokio timeout + // over an uncancellable `spawn_blocking`: a hung cat-file must be REAPED at the + // deadline rather than left to pin the held /ipfs walk permit (#173 round-10, KTD2). + // No outer timeout, mirroring the bounded walk; a `GitServiceTimeout` from either + // twin surfaces as `ServedRead::ReadErr` -> truncated (retryable 503). + let read_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let read = tokio::task::spawn_blocking(move || -> ServedRead { + match store::object_size_bounded(&git_bin, &read_repo, &read_sha, read_timeout) { + Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), + Ok(Some(_)) => {} + // git ran and reported no such object (or an unparseable size): genuine + // not-found for this candidate. + Ok(None) => return ServedRead::Gone, + // git failed to run OR the bounded read timed out (GitServiceTimeout): an + // infra/timeout failure, not a not-found. + Err(e) => return ServedRead::ReadErr(e.to_string()), + } + let content = match store::read_object_content_bounded( + &git_bin, + &read_repo, + &read_sha, + &read_type, + read_timeout, + ) { + Ok(c) => c, + Err(e) => return ServedRead::ReadErr(e.to_string()), + }; + let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); + if served != want_cid { + return ServedRead::Mismatch(served); + } + ServedRead::Ok(content) + }) .await; let served_read = match read { - Ok(Ok(sr)) => sr, - Ok(Err(e)) => { + Ok(sr) => sr, + Err(e) => { tracing::warn!(repo = %repo.name, err = %e, "object read task panicked"); walk.truncated = true; return GateOutcome::Skip; } - Err(_elapsed) => { - tracing::warn!(repo = %repo.name, "object read timed out under the /ipfs walk permit; skipping repo"); - walk.truncated = true; - return GateOutcome::Skip; - } }; let content = match served_read { ServedRead::Ok(c) => c, @@ -1063,38 +1103,33 @@ mod tests { drop(held); } - /// Retain-through-blocking (#174 F5, the load-bearing async property, on the - /// NEWLY-BOUNDED TREE path): the walk admission is held until the `spawn_blocking` - /// walk actually RETURNS, not when a tokio timeout fires. The requested CID - /// resolves to a TREE object under a path-scoped rule, so the gate runs - /// `allowed_tree_set_for_caller_bounded` — the walk this integration converts to - /// `run_bounded_git` — rather than the blob walk #174 already proved. With the - /// global pool at size 1, drive a request until its walk (a fake git that hangs on - /// `rev-list`) is in flight; the slot must stay held (`available_permits() == 0`) - /// and a replacement from a DIFFERENT source must shed 503 for as long as the - /// blocking walk runs — even though the request future is only `.await`ing the - /// blocking join. When the blocking walk ends the permit frees and a replacement - /// is admitted. The permit lives INSIDE the handler across the blocking `.await`; - /// move it out (drop before the walk) and the replacement would be admitted while - /// the walk still burns a blocking thread (the bug this guards). + /// Build the shared `/ipfs` TREE-walk fixture. A fake `git` whose `rev-list` records + /// its pid then sleeps ~6s (so the tree walk blocks deterministically inside + /// `run_bounded_git`) and whose `cat-file -t` answers "tree" (so the bounded + /// object-type probe, `object_type_bounded` on `state.git_bin`, routes into the + /// tree-gate arm); a real SHA-256 bare repo with a committed `src/` tree pinned WITH + /// provenance; and a path-scoped rule so the gate takes the tree-walk branch. Returns + /// the tempdir (keep it alive for the whole test), the state (the caller sets the walk + /// semaphores), the requested CID, and the rev-list pidfile path. #[cfg(unix)] - #[sqlx::test] - async fn get_by_cid_walk_permit_held_through_bounded_tree_walk(pool: sqlx::PgPool) { + async fn seed_tree_walk_fixture( + pool: sqlx::PgPool, + ) -> ( + tempfile::TempDir, + crate::state::AppState, + String, + std::path::PathBuf, + ) { use std::process::Command; let tmp = tempfile::TempDir::new().unwrap(); let revlist_pid = tmp.path().join("revlist.pid"); - // Fake git for the /ipfs TREE walk only (object_type/read_object_content use - // the real `git`, so the tree must genuinely exist below). `rev-parse` - // resolves (so the lenient enumeration appends HEAD) and `rev-list` records - // its pid then sleeps ~6s so the walk BLOCKS deterministically inside - // `run_bounded_git`. The sleep bounds the walk so a broken fix cannot wedge - // the suite. let body = format!( "#!/bin/sh\n\ case \"$1\" in\n\ for-each-ref) : ;;\n\ rev-parse) echo deadbeef ;;\n\ + cat-file) if [ \"$2\" = \"-t\" ]; then echo tree; fi ;;\n\ rev-list) echo $$ > \"{}\"; sleep 6 ;;\n\ *) : ;;\n\ esac\n\ @@ -1114,10 +1149,6 @@ mod tests { let repos_dir = tmp.path().join("repos"); std::fs::create_dir_all(&repos_dir).unwrap(); state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); - // Isolate the global walk pool at size 1; per-source cap permissive so only the - // held global permit can shed the replacement. - state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); - state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); state.git_bin = git_path.to_str().unwrap().to_string(); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; @@ -1129,10 +1160,6 @@ mod tests { .await .unwrap(); let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); - // The exact bare path the handler's `acquire` resolves. Build a REAL SHA-256 - // bare repo there with a committed `src/` directory, so real - // `git cat-file -t ` classifies the requested object as a TREE and the - // handler routes into the tree-walk arm of the gate. let bare = state .repo_store .acquire(&rec.owner_did, &rec.name) @@ -1177,7 +1204,6 @@ mod tests { ], tmp.path(), ); - // The `src` directory's TREE oid — the object the request asks for. let tree_oid = { let out = Command::new("git") .args(["rev-parse", "HEAD:src"]) @@ -1187,8 +1213,6 @@ mod tests { assert!(out.status.success(), "rev-parse failed"); String::from_utf8_lossy(&out.stdout).trim().to_string() }; - // Precondition: real git classifies the object as a TREE (so the handler - // reaches the tree-walk arm, not the blob arm or an early `continue`). assert_eq!( crate::git::store::object_type(&bare, &tree_oid) .unwrap() @@ -1196,9 +1220,6 @@ mod tests { Some("tree"), "the seeded sha256 tree must exist so the handler reaches the tree walk" ); - // Pin the tree's content CID WITH provenance so the resolver targets this one - // repo (no legacy scan). A real pin CID digests the raw object content, not - // the git oid, so build it exactly as the pin path does (#173). let (_ty, raw) = crate::git::store::read_object(&bare, &tree_oid) .unwrap() .expect("tree object readable"); @@ -1208,8 +1229,6 @@ mod tests { .record_pinned_cid(&tree_oid, &cid, Some(&rec.id)) .await .unwrap(); - // A path-scoped rule so has_path_scoped_rule() is true (the tree-gate branch) - // without denying the "/" gate on the public repo. state .db .set_visibility_rule( @@ -1222,6 +1241,34 @@ mod tests { .await .unwrap(); + (tmp, state, cid, revlist_pid) + } + + /// Retain-through-blocking (#174 F5, the load-bearing async property, on the + /// NEWLY-BOUNDED TREE path): the walk admission is held until the `spawn_blocking` + /// walk actually RETURNS, not when a tokio timeout fires. The requested CID + /// resolves to a TREE object under a path-scoped rule, so the gate runs + /// `allowed_tree_set_for_caller_bounded` — the walk this integration converts to + /// `run_bounded_git` — rather than the blob walk #174 already proved. With the + /// global pool at size 1, drive a request until its walk (a fake git that hangs on + /// `rev-list`) is in flight; the slot must stay held (`available_permits() == 0`) + /// and a replacement from a DIFFERENT source must shed 503 for as long as the + /// blocking walk runs — even though the request future is only `.await`ing the + /// blocking join. When the blocking walk ends the permit frees and a replacement + /// is admitted. The permit lives INSIDE the handler across the blocking `.await`; + /// move it out (drop before the walk) and the replacement would be admitted while + /// the walk still burns a blocking thread (the bug this guards). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_permit_held_through_bounded_tree_walk(pool: sqlx::PgPool) { + let (tmp, mut state, cid, revlist_pid) = seed_tree_walk_fixture(pool).await; + // Isolate the global walk pool at size 1; per-source cap permissive so only the + // held global permit can shed the replacement. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + // Keep the fixture tempdir alive for the whole test (its Drop removes the repos). + let _tmp = tmp; + let sem = state.git_ipfs_walk_semaphore.clone(); assert_eq!( sem.available_permits(), @@ -1291,13 +1338,34 @@ mod tests { "a replacement must shed 503 while the prior request's blocking tree walk still runs" ); - // Drop the in-flight request; the detached blocking walk keeps running (a - // spawn_blocking cannot be cancelled), but the permit is a handler local, so - // dropping the future releases it once the blocking join is abandoned. Either - // way, kill the sleeping child so the slot frees promptly and poll for - // recovery — the point already proven above is that the slot stayed held for - // the duration of the blocking work. + // #173 round-10 (R1, KTD1): the NEW invariant. Drop the in-flight request. The + // gated serve pipeline runs in a DETACHED tokio task that OWNS the admission + // guard, so dropping the request future must NOT release admission — the task + // runs to its bounded completion first. Under the OLD handler-local permit the + // slot would free the instant the future dropped even though the blocking walk + // is still burning a thread (the cap bypass this closes). drop(fut); + // While the detached task is still inside the ~6s blocking tree walk, admission + // stays held. Poll a short window: a handler-local permit would already read 1. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + assert_eq!( + sem.available_permits(), + 0, + "dropping the request must NOT release admission while the detached walk \ + still runs (the spawned task owns the guard until its bounded work ends)" + ); + // A replacement from a different source STILL sheds 503 after the drop — the + // detached task holds the global slot, exactly as it did while the future lived. + let peer3: SocketAddr = "203.0.113.83:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer3)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a replacement must still shed 503 after the original request was dropped, \ + while its detached walk holds admission" + ); + // Tear the walk down; the detached task finishes and drops the guard exactly + // once, freeing the single slot back to 1 (never double-freed to 2). unsafe { libc::kill(pid, libc::SIGKILL); } @@ -1311,7 +1379,112 @@ mod tests { } assert!( freed, - "once the blocking walk ends the walk permit must free the global slot" + "once the detached walk tears down, the task drops the guard and frees the slot" + ); + assert_eq!( + sem.available_permits(), + 1, + "admission released exactly once — the single slot is back, not double-freed" + ); + } + + /// Amplification negative (#173 round-10, R1): sequential cancel-spam from ONE source + /// cannot hold more than the per-source cap of concurrent detached walk tasks. A + /// detached serve task holds its per-source permit until its bounded work finishes (up + /// to `git_service_timeout_secs`), so with a per-source cap of 1 a second request from + /// the SAME source sheds 503 even though the GLOBAL pool has room — the source cannot + /// amplify its concurrent walk children past the cap by dropping-and-retrying. (The + /// worst case: a detached task can occupy its global/per-source permit for one + /// bound-interval, so distributed cancel-spam can hold the global pool that long — the + /// accepted bounded-admission tradeoff, not a leak.) + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_cancel_spam_bounded_by_per_source_cap(pool: sqlx::PgPool) { + let (tmp, mut state, cid, revlist_pid) = seed_tree_walk_fixture(pool).await; + // Global pool has ample room (4); the per-source cap is 1. So any shed of a + // same-source replacement is the PER-SOURCE cap, never global exhaustion. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(4)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let _tmp = tmp; + + let sem = state.git_ipfs_walk_semaphore.clone(); + let per_caller = state.git_ipfs_walk_per_caller.clone(); + let router = ipfs_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + // Source S fires request 1; drive until its tree walk is in flight (the task now + // holds source S's single per-source permit). + let source_s: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(source_s))); + let mut walk_pid: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + } + let pid = walk_pid.expect("the fake git rev-list must have spawned"); + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // Cancel-spam: drop request 1's future. Its detached task keeps running the walk + // and KEEPS holding source S's single per-source permit. + drop(fut); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + // A SECOND request from source S sheds 503. The global pool still has room (only 1 + // of 4 taken), so this is the per-source cap, not global exhaustion. + let resp = router.clone().oneshot(make_req(source_s)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a same-source cancel-spam replacement must shed 503 on the per-source cap \ + while the detached task still holds the source's permit" + ); + assert!( + sem.available_permits() >= 3, + "the shed was the per-source cap, not global exhaustion (global pool still has room)" + ); + assert_eq!( + per_caller.tracked_keys(), + 1, + "exactly one per-source permit is outstanding for the one source — no amplification" + ); + + // Tear the detached walk down; the task completes and releases source S's permit + // (tracked_keys returns to 0), so the source is no longer over the cap. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut released = false; + for _ in 0..400 { + if per_caller.tracked_keys() == 0 { + released = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + released, + "once the detached task tears down it releases source S's per-source permit" ); } diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index ec115f1e..50145652 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -1752,8 +1752,9 @@ mod tests { // then, since re-polling a completed future panics. let mut pids = None; for _ in 0..500 { - let finished = - tokio::time::timeout(Duration::from_millis(10), &mut fut).await.is_ok(); + let finished = tokio::time::timeout(Duration::from_millis(10), &mut fut) + .await + .is_ok(); if let Some(p) = read_two_pids(&pidfile) { pids = Some(p); break; diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 25f11246..a79d596a 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -290,26 +290,6 @@ pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> )) } -/// Object size in bytes (`git cat-file -s`) WITHOUT reading the content, so an -/// oversized object can be rejected before it is buffered into memory (#173, F6). -/// `None` if the object does not exist or the size is unparseable. -pub fn object_size(repo_path: &Path, sha256_hex: &str) -> Result> { - // allow-unbounded-git: cheap cat-file -s header read (no content), holds no served-git - // permit and cannot hang; exact twin of object_type above. Not an INV-22 lifecycle op. - let out = Command::new("git") - .args(["cat-file", "-s", sha256_hex]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file -s")?; - if !out.status.success() { - return Ok(None); - } - Ok(String::from_utf8_lossy(&out.stdout) - .trim() - .parse::() - .ok()) -} - /// Read an object's content if its type is already known. pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) -> Result> { let content_output = Command::new("git") @@ -326,6 +306,90 @@ pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) - Ok(content_output.stdout) } +/// Bounded twin of [`object_type`] for the `GET /ipfs/{cid}` serve path (#173 +/// round-10, R1/KTD2). Runs `git cat-file -t` under +/// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git) so the child runs +/// in its own process group and a watchdog reaps it (SIGTERM -> grace -> SIGKILL) at +/// `timeout`. The bare [`object_type`] is a `spawn_blocking` `Command::output` that an +/// async timeout cannot cancel, so a wedged `cat-file` there pins the caller's held +/// /ipfs walk admission for the whole hang; this twin cannot. Semantics mirror +/// [`object_type`]: `Ok(Some(t))` for an existing object, `Ok(None)` when git exits +/// non-zero without timing out (the object is absent), and +/// `Err(`[`GitServiceTimeout`](crate::git::smart_http::GitServiceTimeout)`)` on the +/// deadline so the handler can mark the search truncated (retryable 503) rather than a +/// false not-found. Callers off the /ipfs path keep the bare helper. +pub fn object_type_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + timeout: std::time::Duration, +) -> Result> { + let deadline = std::time::Instant::now() + timeout; + match crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-t", sha256_hex], + repo_path, + b"", + deadline, + ) { + Ok(out) => Ok(Some(String::from_utf8_lossy(&out).trim().to_string())), + Err(e) if e.is::() => Err(e), + // A non-timeout failure is git reporting no such object (a non-zero exit), + // matching `object_type`'s `Ok(None)`. + Err(_) => Ok(None), + } +} + +/// Bounded `git cat-file -s` size read for the `GET /ipfs/{cid}` serve path (#173 +/// round-10, R1/KTD2): reads the object size WITHOUT its content (so an oversized object +/// is rejected before it is buffered, #173 F6), under +/// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git) so a wedged size +/// read is reaped at `timeout` instead of pinning the held /ipfs walk admission. +/// `Ok(Some(n))` on success, `Ok(None)` when the object is absent (a non-timeout +/// non-zero exit), `Err(GitServiceTimeout)` on the deadline. +pub fn object_size_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + timeout: std::time::Duration, +) -> Result> { + let deadline = std::time::Instant::now() + timeout; + match crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-s", sha256_hex], + repo_path, + b"", + deadline, + ) { + Ok(out) => Ok(String::from_utf8_lossy(&out).trim().parse::().ok()), + Err(e) if e.is::() => Err(e), + Err(_) => Ok(None), + } +} + +/// Bounded twin of [`read_object_content`] for the `GET /ipfs/{cid}` serve path (#173 +/// round-10, R1/KTD2): `git cat-file ` under +/// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git) so a wedged content +/// read is reaped at `timeout` instead of pinning the held /ipfs walk admission. Returns +/// the raw object bytes on success and an error (including `GitServiceTimeout` on the +/// deadline) otherwise, mirroring [`read_object_content`]. +pub fn read_object_content_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + obj_type: &str, + timeout: std::time::Duration, +) -> Result> { + let deadline = std::time::Instant::now() + timeout; + crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", obj_type, sha256_hex], + repo_path, + b"", + deadline, + ) +} + /// Read a git object by its SHA-256 hex object ID. /// /// Returns `(object_type, content_bytes)` where `content_bytes` is the raw @@ -521,4 +585,84 @@ mod tests { "unchanged file must not appear: {names:?}" ); } + + /// #173 round-10 (KTD2): `object_type_bounded` reaps a wedged `cat-file` child at its + /// deadline instead of blocking on it to natural exit, so a hung probe cannot pin the + /// /ipfs walk admission the owning task holds. A fake `git` records its pid and sleeps + /// far past the 1s deadline; the `run_bounded_git` watchdog (SIGTERM -> grace -> + /// SIGKILL of the process group) must kill it well before that natural exit, and the + /// call must surface `GitServiceTimeout`. REVERT PROOF (RED): swap the twin's + /// `run_bounded_git` for the bare `Command::output()` and the wedged child stays alive + /// past the deadline — the mid-flight liveness poll below reads it still running. + #[cfg(unix)] + #[test] + fn object_type_bounded_reaps_wedged_child_at_deadline() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("catfile.pid"); + // `cat-file` records its own pid then sleeps 8s (>> the 1s deadline) so the probe + // is genuinely wedged; the watchdog is what must end it. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + cat-file) echo $$ > \"{}\"; sleep 8 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pidfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let repo = tmp.path().to_path_buf(); + let git = git_path.to_str().unwrap().to_string(); + + let alive = |pid: i32| unsafe { libc::kill(pid, 0) == 0 }; + + // The bounded probe blocks until the watchdog tears the child down, so run it on + // a worker thread and poll for the reap from here. + let handle = std::thread::spawn(move || { + super::object_type_bounded(&git, &repo, "deadbeef", Duration::from_secs(1)) + }); + + let mut pid = None; + for _ in 0..500 { + if let Some(p) = std::fs::read_to_string(&pidfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + pid = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let pid = pid.expect("the fake cat-file must have spawned and recorded its pid"); + + // Past the 1s deadline + SIGTERM grace but well before the 8s natural exit: the + // watchdog must already have reaped the wedged group. A bare, unbounded read would + // leave it running here — the load-bearing RED. + std::thread::sleep(Duration::from_secs(3)); + let reaped = !alive(pid); + // Defensive reap so a RED run leaks no orphan. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + assert!( + reaped, + "object_type_bounded must reap the wedged cat-file child at the deadline, \ + not leave it running to its natural exit" + ); + + let res = handle.join().expect("probe thread joins"); + let err = res.expect_err("a deadline overrun must be an error, not a value"); + assert!( + err.is::(), + "a deadline overrun must surface GitServiceTimeout, got: {err:?}" + ); + } } diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index e750a48f..5ec653fc 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -29,6 +29,7 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { let repos = src("api/repos.rs"); let smart_http = src("git/smart_http.rs"); let vis = src("git/visibility_pack.rs"); + let ipfs = src("api/ipfs.rs"); // U1 / P1-a: run_bounded_git stands the watchdog down only after confirming the // child actually terminated (WNOWAIT), not on the raw stdout-drain EOF — otherwise @@ -84,6 +85,38 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { is reaped on disconnect (the path-scoped half of #174 P1-a)" ); + // U2 / R1 (#173 round-10): the `GET /ipfs/{cid}` serve pipeline must run in a + // DETACHED tokio task that OWNS the AdmissionGuard built from BOTH walk permits, so a + // cancelled or timed-out request releases admission only after the spawned (bounded) + // work completes — not the instant the handler future drops (the /ipfs half of #174 + // P1-a). Two load-bearing markers: the guard is constructed from the two named + // permits, and the whole pipeline is moved into a `tokio::spawn`. Reverting to + // handler-local permits (dropping the guard/spawn) trips this. + assert!( + ipfs.contains("AdmissionGuard::new(ipfs_walk_permit, ipfs_caller_permit)") + && ipfs.contains( + "let serve: tokio::task::JoinHandle> = tokio::spawn(async move {" + ), + "U2/R1 gate missing: get_by_cid must move both /ipfs admission permits into an \ + AdmissionGuard owned by a detached tokio::spawn task so admission is released \ + only after the spawned serve work completes (the /ipfs half of #174 P1-a)" + ); + + // U2 / KTD2 (#173 round-10): the probe/read children on the /ipfs path must be the + // duration-bounded twins (process-group teardown via run_bounded_git), not the bare + // `store::object_type` / `read_object_content` (or an unbounded `cat-file -s`) a tokio + // timeout cannot cancel — otherwise a wedged cat-file lingers and pins the held + // admission past the deadline. Reverting any twin call site back to a bare read trips + // this. + assert!( + ipfs.contains("object_type_bounded(") + && ipfs.contains("object_size_bounded(") + && ipfs.contains("read_object_content_bounded("), + "U2/KTD2 gate missing: the /ipfs probe+read must call the run_bounded_git-backed \ + *_bounded twins so a wedged cat-file is reaped at the deadline, not left to pin \ + the held /ipfs walk admission" + ); + // P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere // but inside withheld_recipients_gated. A second call site (count > 1) is a new // detached git walk that skips the admission gate — exactly the class U5 closed. From 6222775bb12f13736cb7caf8af5ad23955066ef7 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 20 Jul 2026 19:04:55 -0500 Subject: [PATCH 09/77] fix(node): requeue coalesced pushes instead of dropping them Per-repo coalescing dropped every push that arrived while a repo's encryption task was in flight, and no reconciliation ever processed it, so a withheld blob added by the coalesced push had its recovery copy permanently absent. The drop also silently skipped the coalesced push's local-IPFS pin work. EncryptInflight becomes a dirty-flag map; the detached task loops, and the atomic check-and-clear sits at the task tail, unconditional on the has_path_scoped_rule gate and walk success (a public or rules-free repo would otherwise exit before it ran). Each requeue re-reads repo state fresh and re-enumerates the pin half through the fail-closed full scan, never bare list_all_objects, so a coalesced rule change is honored and no withheld or dangling object leaks. At-most-one-task-per-repo is preserved. --- crates/gitlawb-node/src/api/repos.rs | 531 +++++++++++++++++++----- crates/gitlawb-node/src/state.rs | 124 ++++-- crates/gitlawb-node/src/test_support.rs | 351 ++++++++++++++++ 3 files changed, 857 insertions(+), 149 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index fb902f01..c9a2b3dd 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -790,6 +790,269 @@ async fn withheld_recipients_gated( .await } +/// Everything the detached post-push replication task needs that does not change +/// between requeue passes. Cloned once from `AppState` at the spawn site so the task +/// is self-contained (the handler keeps no copy). +struct PostPushReplication { + db: std::sync::Arc, + disk_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, + ipfs_api: String, + repo_id: String, + encrypt_sem: std::sync::Arc, + node_seed: [u8; 32], + node_did: String, + repo_name: String, + irys_url: String, + http_client: std::sync::Arc, +} + +/// The requeue enumeration for the pin half: a fail-closed FULL scan of the current +/// object DB under the REFRESHED rules. The coalesced push's ref tips are gone by the +/// time we requeue, so the delta path is unavailable; the whole-repo scan is the safe +/// superset. Never pins the bare `list_all_objects` output — that includes +/// dangling/withheld blobs — it feeds it as CANDIDATES to the same fail-closed filter +/// the push path's full-scan branch uses, which drops dangling and visibility-withheld +/// blobs before anything is pinned. +async fn requeue_full_scan_object_list( + disk_path: &std::path::Path, + git_bin: &str, + timeout: std::time::Duration, + rules: Vec, + is_public: bool, + owner_did: String, +) -> Vec { + let disk = disk_path.to_path_buf(); + let gb = git_bin.to_string(); + let candidates = tokio::task::spawn_blocking(move || { + crate::git::push_delta::list_all_objects(&disk, &gb, std::time::Instant::now() + timeout) + }) + .await + .ok() + .and_then(|r| { + r.map_err(|e| { + tracing::warn!(err = %e, "requeue full-scan enumeration failed; pinning nothing this pass") + }) + .ok() + }) + .unwrap_or_default(); + fail_closed_full_scan_objects( + disk_path.to_path_buf(), + rules, + is_public, + owner_did, + candidates, + git_bin.to_string(), + timeout, + ) + .await +} + +/// The detached post-push encryption + local-IPFS pin task, as a REQUEUE LOOP. +/// +/// Pass one uses the spawn-time captures (`first_*`) — the delta the push handler +/// already computed. At the TASK TAIL (unconditional on the encrypt gate below and on +/// walk success) it consults the coalescing dirty flag: if a push coalesced during the +/// window it re-reads repo state (rules, is_public, owner_did, withheld) FRESH from the +/// DB, re-enumerates the pin set fail-closed, and runs another pass — so the coalesced +/// push is covered before the task exits. Otherwise it releases the key and returns. +/// +/// The tail placement is load-bearing: the encrypt+anchor block only runs under a +/// path-scoped rule, so a check-and-clear placed inside that gate would never run for a +/// public/rules-free repo or a failed walk, dropping exactly the pin-half push this +/// requeue must cover. +#[allow(clippy::too_many_arguments)] +async fn run_post_push_replication( + mut guard: crate::state::EncryptInflightGuard, + ctx: PostPushReplication, + first_object_list: Vec, + first_rules: Option>, + first_is_public: bool, + first_owner_did: String, + first_withheld: std::collections::HashSet, +) { + let mut object_list = first_object_list; + let mut rules_opt = first_rules; + let mut is_public = first_is_public; + let mut owner_did = first_owner_did; + // The task only spawns when `withheld.is_some()`, so pass one always replicates. + let mut withheld: Option> = Some(first_withheld); + + loop { + if withheld.is_some() { + // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). + crate::ipfs_pin::pin_new_objects( + &ctx.ipfs_api, + &ctx.disk_path, + object_list.clone(), + &ctx.db, + &ctx.repo_id, + ) + .await; + + // Option B1: encrypt-then-pin the withheld blobs. No path-scoped rule can + // withhold a blob, so a rules-free repo has nothing to seal; skip. Mirrors + // the has_path_scoped_rule gate on the other two withheld-walk sites. + if let Some(rules) = rules_opt + .clone() + .filter(|r| visibility_pack::has_path_scoped_rule(r)) + { + let recip = withheld_recipients_gated( + ctx.encrypt_sem.clone(), + ctx.disk_path.clone(), + ctx.git_bin.clone(), + ctx.timeout, + rules, + is_public, + owner_did.clone(), + ) + .await; + if let Ok(Ok(recipients)) = recip { + let delta = crate::encrypted_pin::encrypt_and_pin( + &ctx.ipfs_api, + &ctx.disk_path, + &ctx.db, + &ctx.repo_id, + &ctx.node_seed, + &recipients, + ) + .await; + + // Option B3: anchor a per-push manifest of the sealed blobs to + // Arweave. Best-effort; never fails the push. + if !delta.is_empty() && !ctx.irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&owner_did); + let repo_slug = format!("{owner_short}/{}", ctx.repo_name); + let ts = chrono::Utc::now().to_rfc3339(); + let manifest = crate::arweave::EncryptedManifest { + repo: &repo_slug, + owner_did: &owner_did, + node_did: &ctx.node_did, + timestamp: &ts, + blobs: &delta, + }; + match crate::arweave::anchor_encrypted_manifest( + &ctx.http_client, + &ctx.irys_url, + &manifest, + ) + .await + { + Ok(tx) if !tx.is_empty() => tracing::info!( + repo = %repo_slug, + tx_id = %tx, + "anchored encrypted manifest to Arweave" + ), + Ok(_) => {} + Err(e) => tracing::warn!( + repo = %repo_slug, + err = %e, + "encrypted manifest anchor failed" + ), + } + } + } + } + } + + // TASK TAIL — unconditional check-and-clear, atomic with the release decision. + if !guard.requeue_or_release() { + break; + } + + // A push coalesced during this pass. Re-read repo state FRESH (never the stale + // spawn-time captures) so a coalesced push that changed `.gitlawb` withholding + // is walked under the new policy, then re-enumerate the pin set fail-closed. + let (r_rules, r_is_public, r_owner) = match ctx.db.get_repo_by_id(&ctx.repo_id).await { + Ok(Some(rec)) => ( + ctx.db.list_visibility_rules(&ctx.repo_id).await.ok(), + rec.is_public, + rec.owner_did, + ), + Ok(None) => { + tracing::debug!(repo = %ctx.repo_id, "repo gone before requeue pass; releasing"); + (None, false, String::new()) + } + Err(e) => { + tracing::warn!(repo = %ctx.repo_id, err = %e, "requeue repo re-read failed; skipping this pass's work"); + (None, false, String::new()) + } + }; + let (_announce, r_withheld) = replication_withheld_set( + r_rules.clone(), + &r_owner, + r_is_public, + ctx.disk_path.clone(), + ctx.git_bin.clone(), + ctx.timeout, + ) + .await; + object_list = match &r_withheld { + Some(_) => { + requeue_full_scan_object_list( + &ctx.disk_path, + &ctx.git_bin, + ctx.timeout, + r_rules.clone().unwrap_or_default(), + r_is_public, + r_owner.clone(), + ) + .await + } + None => Vec::new(), + }; + rules_opt = r_rules; + is_public = r_is_public; + owner_did = r_owner; + withheld = r_withheld; + } +} + +/// Test-only entry point: build the `PostPushReplication` context from a test +/// `AppState` (with an overridable `ipfs_api` for a mock Kubo server and an explicit +/// `disk_path` for the fixture repo) and run the requeue loop. Keeps +/// `PostPushReplication` and `run_post_push_replication` private to this module. +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +pub(crate) async fn run_post_push_replication_for_test( + state: &AppState, + guard: crate::state::EncryptInflightGuard, + disk_path: std::path::PathBuf, + repo_id: String, + ipfs_api: String, + is_public: bool, + owner_did: String, + object_list: Vec, + rules: Option>, + withheld: std::collections::HashSet, +) { + let ctx = PostPushReplication { + db: state.db.clone(), + disk_path: disk_path.clone(), + git_bin: state.git_bin.clone(), + timeout: std::time::Duration::from_secs(state.config.git_service_timeout_secs), + ipfs_api, + repo_id, + encrypt_sem: state.git_encrypt_semaphore.clone(), + node_seed: *state.node_keypair.to_seed(), + node_did: state.node_did.to_string(), + repo_name: String::new(), + irys_url: String::new(), + http_client: std::sync::Arc::clone(&state.http_client), + }; + run_post_push_replication( + guard, + ctx, + object_list, + rules, + is_public, + owner_did, + withheld, + ) + .await; +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -1434,128 +1697,44 @@ pub async fn git_receive_pack( // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). // Skipped entirely when the public cannot read the repo (withheld == None). // - // Coalesce per repo (#174 P2-2): this task parks on `git_encrypt_semaphore` - // (which DEFERS when the pool is full rather than dropping the recovery copy). To - // bound the OUTSTANDING parked-task set, only spawn if no encryption task for this - // repo is already in flight; otherwise skip — the pending/next walk over this - // repo's history already covers the newer push's objects, so the recovery copy is - // delayed, never lost. The guard removes the repo key when the task ends (success, - // error, or panic), so a later push is re-admitted (no permanent skip). - if withheld.is_some() { + // Coalesce per repo (#174 P2-2): only spawn a task if none is in flight for this + // repo; otherwise `try_begin` marks the repo DIRTY and the in-flight task requeues + // one more pass (re-reading fresh repo state) before it exits. This bounds the + // outstanding parked-task set to one per repo while covering every coalesced push — + // there is no reconciliation sweep, so a dropped job would be lost forever (#173 F3). + if let Some(withheld_set) = withheld.clone() { match state.encrypt_inflight.try_begin(&record.id) { None => { tracing::debug!( repo = %record.id, "post-push encryption task already in flight for this repo; coalescing \ - (the pending recovery-copy walk covers this push's objects)" + (the in-flight task will requeue one more pass to cover this push)" ); } Some(inflight_guard) => { - let object_list_ipfs = object_list.clone(); - let ipfs_api = state.config.ipfs_api.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let rules_for_enc = rules_opt.clone(); - let repo_id = record.id.clone(); - let owner_did = record.owner_did.clone(); - let is_public = record.is_public; - let irys_url = state.config.irys_url.clone(); - let http_client = std::sync::Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let node_seed = state.node_keypair.to_seed(); - let repo_name = record.name.clone(); - let enc_git_bin = state.git_bin.clone(); - let enc_timeout = - std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let encrypt_sem = state.git_encrypt_semaphore.clone(); - tokio::spawn(async move { - // Held for the whole task; drop (on completion/error/panic) releases the - // repo's coalescing key so the next push for this repo can spawn again. - let _inflight_guard = inflight_guard; - let pinned = crate::ipfs_pin::pin_new_objects( - &ipfs_api, - &repo_path_clone, - object_list_ipfs, - &db_clone, - &repo_id, - ) - .await; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); - for (sha, cid) in &pinned { - tracing::info!(sha = %sha, %cid, "pinned"); - } - } - - // Option B1: encrypt-then-pin the withheld blobs so authorized - // readers can recover them when the origin cannot serve them. - // No path-scoped rule can withhold a blob, so withheld_blob_recipients - // would return an empty map after a full per-ref walk; skip it. Mirrors - // the has_path_scoped_rule gate on the other two withheld-walk sites. - if let Some(rules) = - rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) - { - // Bound the number of concurrent post-push encryption walks (#174 P1-e): - // acquire an admission permit before the full-history walk, deferring - // when the pool is full rather than shedding the recovery pin. - let recip = withheld_recipients_gated( - encrypt_sem.clone(), - repo_path_clone.clone(), - enc_git_bin.clone(), - enc_timeout, - rules, - is_public, - owner_did.clone(), - ) - .await; - if let Ok(Ok(recipients)) = recip { - let delta = crate::encrypted_pin::encrypt_and_pin( - &ipfs_api, - &repo_path_clone, - &db_clone, - &repo_id, - &node_seed, - &recipients, - ) - .await; - - // Option B3: anchor a per-push manifest of the blobs sealed - // this push to Arweave, so the oid->cid index survives total - // node loss. Best-effort; never fails the push. - if !delta.is_empty() && !irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&owner_did); - let repo_slug = format!("{owner_short}/{repo_name}"); - let ts = chrono::Utc::now().to_rfc3339(); - let manifest = crate::arweave::EncryptedManifest { - repo: &repo_slug, - owner_did: &owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &delta, - }; - match crate::arweave::anchor_encrypted_manifest( - &http_client, - &irys_url, - &manifest, - ) - .await - { - Ok(tx) if !tx.is_empty() => tracing::info!( - repo = %repo_slug, - tx_id = %tx, - "anchored encrypted manifest to Arweave" - ), - Ok(_) => {} - Err(e) => tracing::warn!( - repo = %repo_slug, - err = %e, - "encrypted manifest anchor failed" - ), - } - } - } - } - }); + let ctx = PostPushReplication { + db: state.db.clone(), + disk_path: disk_path.clone(), + git_bin: state.git_bin.clone(), + timeout: std::time::Duration::from_secs(state.config.git_service_timeout_secs), + ipfs_api: state.config.ipfs_api.clone(), + repo_id: record.id.clone(), + encrypt_sem: state.git_encrypt_semaphore.clone(), + node_seed: *state.node_keypair.to_seed(), + node_did: state.node_did.to_string(), + repo_name: record.name.clone(), + irys_url: state.config.irys_url.clone(), + http_client: std::sync::Arc::clone(&state.http_client), + }; + tokio::spawn(run_post_push_replication( + inflight_guard, + ctx, + object_list.clone(), + rules_opt.clone(), + record.is_public, + record.owner_did.clone(), + withheld_set, + )); } } } @@ -4573,6 +4752,130 @@ mod tests { ); } + // ---- U3 (#173 F3): dirty-flag requeue seam (the mechanics behind the end-to-end + // requeue tests in test_support.rs) ---- + + /// A coalesced push MARKS the repo dirty (not just "skip"), and the in-flight task's + /// tail check-and-clear then runs ONE more pass before releasing: `requeue_or_release` + /// returns `true` (loop) while dirty, clearing the flag, then `false` (release) when + /// clean. This is the mechanism that makes a coalesced push requeued, not dropped. + #[test] + fn u3_try_begin_marks_dirty_then_requeue_loops_once_then_releases() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkU3DirtyAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/proj"; + + let mut guard = inflight.try_begin(repo).expect("first push admits"); + assert_eq!(inflight.dirty(repo), Some(false), "fresh task starts clean"); + + // A push coalesces during the in-flight window: marked dirty, no new task. + assert!( + inflight.try_begin(repo).is_none(), + "coalesced push does not spawn" + ); + assert_eq!( + inflight.dirty(repo), + Some(true), + "the coalesced push marked the repo dirty" + ); + + // Task tail, pass 1: dirty -> requeue (clear the flag, keep the key, loop). + assert!( + guard.requeue_or_release(), + "dirty repo requeues one more pass" + ); + assert_eq!( + inflight.dirty(repo), + Some(false), + "the dirty flag is cleared for the next pass" + ); + assert_eq!( + inflight.len(), + 1, + "the key is still held while the task loops" + ); + + // Task tail, pass 2: clean -> release (remove the key, exit). + assert!( + !guard.requeue_or_release(), + "a clean repo releases and exits" + ); + assert_eq!(inflight.len(), 0, "the key is removed on the clean release"); + assert_eq!( + inflight.dirty(repo), + None, + "no in-flight entry after release" + ); + } + + /// The check-and-clear is ATOMIC with the release decision, so no push lands in a + /// "checked clean but still present" gap (scenario 6, race gap). A push that arrives + /// BEFORE the tail check sets dirty -> the same pass requeues it. A push that arrives + /// AFTER a clean release finds an empty set -> `try_begin` spawns a fresh task. Both + /// directions covered; neither drops the push. + #[test] + fn u3_requeue_or_release_leaves_no_uncovered_gap() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkU3GapBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB/proj"; + + // Case A: push arrives before the tail check -> requeue covers it. + let mut guard = inflight.try_begin(repo).expect("admit"); + assert!( + inflight.try_begin(repo).is_none(), + "push arrives during the window" + ); + assert!( + guard.requeue_or_release(), + "the pre-check push is covered by a requeue" + ); + // Now clean: the task releases and exits. + assert!(!guard.requeue_or_release(), "clean -> release"); + assert_eq!(inflight.len(), 0); + + // Case B: a push arriving after release starts a brand-new task (not dropped). + let guard2 = inflight.try_begin(repo); + assert!( + guard2.is_some(), + "a post-release push spawns a fresh task, never dropped" + ); + } + + /// Drop is a PANIC BACKSTOP only. A task that panics before releasing still frees the + /// key (so a crashed walk never permanently locks the repo out); a task that releases + /// via `requeue_or_release` and then drops does not double-free. + #[test] + fn u3_drop_is_panic_backstop_for_unreleased_guard() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkU3PanicCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC/proj"; + + // Normal release then drop: key already gone, drop is a no-op. + let mut g = inflight.try_begin(repo).expect("admit"); + assert!(!g.requeue_or_release(), "clean release"); + assert_eq!(inflight.len(), 0); + drop(g); + assert_eq!( + inflight.len(), + 0, + "dropping an already-released guard does not resurrect a key" + ); + + // Panic before releasing: Drop-on-unwind frees the key. + let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _g = inflight.try_begin(repo).expect("admit before panic"); + assert_eq!(inflight.len(), 1); + panic!("task panics before reaching its tail check-and-clear"); + })); + assert!(panicked.is_err(), "the simulated task panicked"); + assert_eq!( + inflight.len(), + 0, + "a panic before release still frees the key (backstop), so the repo is not locked out" + ); + assert!( + inflight.try_begin(repo).is_some(), + "the repo can be admitted again after a panic" + ); + } + /// Model of the pre-fix / mutated code: no coalescing check, so every push spawns. /// Returns the count of tasks spawned (== the size of the unbounded outstanding set /// the fix prevents), used as the RED comparison in the bound test above. diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 10dbbf32..61bbec6a 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -253,24 +253,29 @@ impl AppState { /// repo spawn N parked tasks, each holding cloned object lists/rules/paths/keys — an /// unbounded outstanding set. /// -/// This tracks the repo keys with an in-flight encryption task. Before spawning, the -/// handler calls [`try_begin`](Self::try_begin): if a task for the repo is already -/// in-flight it returns `None` and the handler SKIPS spawning a duplicate (coalesce — -/// the newer push's objects are covered by the pending/next walk over the same repo's -/// history). This bounds the outstanding set to <=1 pending task per repo WITHOUT -/// dropping work: coalescing only delays a duplicate walk, it never sheds the recovery -/// copy (there is no reconciliation sweep, so a *dropped* job would be lost forever). +/// This tracks the repo keys with an in-flight encryption task, each carrying a +/// DIRTY flag. Before spawning, the handler calls [`try_begin`](Self::try_begin): if +/// a task for the repo is already in-flight it MARKS THE REPO DIRTY and returns `None` +/// so the handler SKIPS spawning a duplicate (coalesce). The in-flight task consults +/// the flag at its tail via [`requeue_or_release`](EncryptInflightGuard::requeue_or_release): +/// a set flag makes it run ONE MORE pass (re-reading repo state) before it exits, so a +/// push coalesced during the in-flight window is REQUEUED, never dropped. This bounds +/// the outstanding set to <=1 task per repo without losing work: there is no +/// reconciliation sweep, so a *dropped* job would be lost forever. /// -/// The returned [`EncryptInflightGuard`] is moved into the detached task and removes -/// the repo key on drop — on normal completion, error, OR panic (Drop runs on unwind) -/// — so one crashed walk can never permanently lock a repo out of future recovery -/// copies. +/// The returned [`EncryptInflightGuard`] is moved into the detached task. The normal +/// exit path is `requeue_or_release`, which removes the repo key ATOMICALLY with the +/// "clean" decision — no push can land in the gap between "checked clean" and "task +/// exits". `Drop` is a panic backstop only: if the task panics (or returns without +/// calling `requeue_or_release`) it still removes the key so a crashed walk can never +/// permanently lock a repo out of future recovery copies. #[derive(Clone, Default)] pub struct EncryptInflight { - // std::sync::Mutex: only ever held for O(1) HashSet insert/remove in a sync - // context (right before `tokio::spawn`, and in the guard's Drop) — never across - // an await, so a std Mutex is correct and cheaper than a tokio one. - repos: Arc>>, + // std::sync::Mutex: only ever held for O(1) HashMap ops in a sync context (before + // `tokio::spawn`, at the task tail, and in Drop) — never across an await, so a std + // Mutex is correct and cheaper than a tokio one. The value is the DIRTY flag: a + // coalesced push flips it true so the in-flight task requeues one more pass. + repos: Arc>>, } impl EncryptInflight { @@ -279,18 +284,24 @@ impl EncryptInflight { } /// Try to begin an encryption task for `repo_id`. Returns `Some(guard)` if no task - /// for the repo was in-flight (the caller should spawn), or `None` if one already - /// is (the caller should COALESCE — skip spawning a duplicate). The guard releases - /// the repo key on drop. + /// for the repo was in-flight (the caller should spawn). If one already is, MARKS + /// the repo dirty and returns `None` (the caller COALESCES — skips the duplicate + /// spawn; the in-flight task requeues one more pass to cover this push). pub fn try_begin(&self, repo_id: &str) -> Option { - let mut set = self.repos.lock().expect("encrypt_inflight mutex poisoned"); - if set.insert(repo_id.to_string()) { - Some(EncryptInflightGuard { - repos: Arc::clone(&self.repos), - repo_id: repo_id.to_string(), - }) - } else { - None + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.get_mut(repo_id) { + Some(dirty) => { + *dirty = true; + None + } + None => { + map.insert(repo_id.to_string(), false); + Some(EncryptInflightGuard { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + released: false, + }) + } } } @@ -309,23 +320,66 @@ impl EncryptInflight { pub fn is_empty(&self) -> bool { self.len() == 0 } + + /// Test-only: read the dirty flag for `repo_id` (`None` if no task is in-flight). + #[cfg(test)] + pub fn dirty(&self, repo_id: &str) -> Option { + self.repos + .lock() + .expect("encrypt_inflight mutex poisoned") + .get(repo_id) + .copied() + } } -/// RAII guard removing a repo key from [`EncryptInflight`] when the detached -/// encryption task finishes (drop on completion, error, or panic-unwind). Move-only — -/// there is no reason to clone a guard, and cloning would double-remove. +/// RAII guard for one in-flight encryption task's repo key. The task drives it at its +/// tail with [`requeue_or_release`](Self::requeue_or_release); `Drop` is a panic +/// backstop. Move-only — cloning would double-release. pub struct EncryptInflightGuard { - repos: Arc>>, + repos: Arc>>, repo_id: String, + released: bool, +} + +impl EncryptInflightGuard { + /// TASK-TAIL check-and-clear, atomic with the release decision. If a push + /// coalesced since the last pass (dirty), clear the flag and return `true` (the + /// task loops one more pass). Otherwise remove the key and return `false` (the task + /// exits). Atomic under the mutex: a concurrent push either sets the flag BEFORE + /// this reads it (-> requeue covers it) or arrives AFTER the key is removed (-> a + /// fresh `try_begin` spawns a new task) — there is no window where the key is + /// present-but-clean while the task exits. + pub fn requeue_or_release(&mut self) -> bool { + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.get_mut(&self.repo_id) { + Some(dirty) if *dirty => { + *dirty = false; + true + } + _ => { + map.remove(&self.repo_id); + self.released = true; + false + } + } + } } impl Drop for EncryptInflightGuard { fn drop(&mut self) { - // A poisoned lock (a prior panic while holding it) still lets us take the inner - // set via into_inner-on-guard; but poisoning here is not expected because the - // only critical sections are the O(1) ops above. Remove best-effort. - if let Ok(mut set) = self.repos.lock() { - set.remove(&self.repo_id); + // Panic backstop ONLY. The normal exit is `requeue_or_release` (which removed + // the key atomically and set `released`), so this does nothing then. If the + // task panicked or returned without releasing, remove the key so one crashed + // walk never permanently locks the repo out. + // + // Accepted residual: a panic with the dirty flag still set drops the requeued + // work — Drop cannot loop — the same loss class as the on-panic behavior before + // this change. No reconciliation sweep re-derives it. + if self.released { + return; + } + if let Ok(mut map) = self.repos.lock() { + map.remove(&self.repo_id); } } } diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 45b2b807..20e325ac 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -7306,4 +7306,355 @@ mod tests { "result includes the deep cert matching the prefix" ); } + + // ---- U3 (#173 F3): coalesced post-push work is REQUEUED, not dropped ---- + // + // The seam mechanics (dirty flag, atomic check-and-clear, Drop backstop) are unit + // tested next to the #174 coalescing tests in `api/repos.rs`. These drive the whole + // detached task through a mock Kubo node and a real git repo, so the requeue's + // fresh re-read (encrypt half) and fail-closed full-scan enumeration (pin half) are + // proven end to end, with DB-observable effects. Each test models a push that + // coalesced during the in-flight window by (a) marking the repo dirty via a second + // `try_begin` and (b) making the repo/policy dynamic so the FIRST pass's spawn-time + // captures are stale — a static-state test would pass vacuously over the gap. + mod u3_requeue { + use super::*; + use crate::db::VisibilityMode; + use std::collections::HashSet; + use std::path::{Path, PathBuf}; + use std::process::Command; + + fn git(args: &[&str], dir: &Path) { + let ok = Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(); + assert!(ok, "git {args:?} failed"); + } + fn oid(rev: &str, dir: &Path) -> String { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(dir) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}: {out:?}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + struct Repo { + _td: tempfile::TempDir, + path: PathBuf, + } + fn init_repo() -> Repo { + let td = tempfile::TempDir::new().unwrap(); + let path = td.path().to_path_buf(); + git(&["init", "-q"], &path); + git(&["config", "user.email", "t@t"], &path); + git(&["config", "user.name", "t"], &path); + Repo { _td: td, path } + } + /// Commit `content` at `rel`, return the blob oid. + fn commit(repo: &Path, rel: &str, content: &str) -> String { + let full = repo.join(rel); + std::fs::create_dir_all(full.parent().unwrap()).unwrap(); + std::fs::write(&full, content).unwrap(); + git(&["add", "."], repo); + git(&["commit", "-qm", rel], repo); + oid(&format!("HEAD:{rel}"), repo) + } + /// Write a loose, UNREACHABLE blob (dangling object). + fn write_dangling_blob(repo: &Path, content: &str) -> String { + let out = Command::new("git") + .args(["hash-object", "-w", "--stdin"]) + .current_dir(repo) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + use std::io::Write; + out.stdin + .as_ref() + .unwrap() + .write_all(content.as_bytes()) + .unwrap(); + let o = out.wait_with_output().unwrap(); + assert!(o.status.success()); + String::from_utf8_lossy(&o.stdout).trim().to_string() + } + fn new_did() -> String { + Keypair::generate().did().to_string() + } + + /// SCENARIO 2 + 5 (pin half, TAIL-PLACEMENT guard). A coalesced push on a PUBLIC + /// repo with NO path-scoped rule must still requeue its pin half: the second + /// push's new object is pinned after the task. RED without the loop (stale spawn + /// object_list never lists obj2), and RED if the check-and-clear sits inside the + /// `has_path_scoped_rule` block (a rules-free repo would never reach it). + #[sqlx::test] + async fn u3_rules_free_public_repo_requeues_pin_half(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-pin"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + // The coalesced push B adds obj2 (present at requeue time, NOT in the stale + // push-A spawn object_list). + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Push A admits (guard); push B coalesces (marks dirty). + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces while A is in flight" + ); + + // Spawn-time (push A) captures are STALE: object_list lists only obj1, no rule. + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj1.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "push A's object is pinned on the first pass" + ); + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's new object is pinned by the REQUEUE full scan (RED \ + without the loop, or if the check-and-clear sits inside the encrypt gate)" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits clean" + ); + } + + /// SCENARIO 1 + 3 (encrypt half, FRESH re-read). A coalesced push adds a + /// path-scoped rule withholding a blob. The task must re-read rules FRESH on + /// requeue and seal the newly-withheld blob's recovery copy. RED without the loop + /// (pass one's stale empty rule set seals nothing). + #[sqlx::test] + async fn u3_requeue_seals_blob_withheld_by_coalesced_rule_change(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-enc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let _pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + + // Coalesced push B changes .gitlawb: withhold /secret/** from anon, grant reader. + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + // Push A captures are STALE: no rule, empty withheld set (public repo). + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "the coalesced push's newly-withheld blob is sealed after the REQUEUE re-read \ + (RED without the loop: pass one's stale empty rules seal nothing)" + ); + assert!(state.encrypt_inflight.is_empty(), "guard key released"); + } + + /// SCENARIO 4 (visibility-leak negative). The requeue full scan must feed + /// `list_all_objects` through the fail-closed filter, never pin it bare: a + /// withheld secret blob and a dangling blob must NOT land in the public pin set. + #[sqlx::test] + async fn u3_requeue_full_scan_does_not_publicly_pin_withheld_or_dangling(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-leak"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + // Coalesced push adds a new public object and a dangling blob. + let new_pub_oid = commit(&git_repo.path, "public/c.txt", "more public\n"); + let dangling_oid = write_dangling_blob(&git_repo.path, "orphan bytes\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let rules = state.db.list_visibility_rules(&repo.id).await.unwrap(); + let mut withheld = HashSet::new(); + withheld.insert(secret_oid.clone()); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![pub_oid.clone()], + Some(rules), + withheld, + ) + .await; + + assert!( + state.db.is_pinned(&new_pub_oid).await.unwrap(), + "the coalesced push's new PUBLIC object is pinned by the requeue" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "a WITHHELD blob is never publicly pinned by the requeue enumeration (leak guard)" + ); + assert!( + !state.db.is_pinned(&dangling_oid).await.unwrap(), + "a DANGLING blob is never publicly pinned by the requeue enumeration (leak guard)" + ); + // The withheld blob still gets its ENCRYPTED recovery copy (not a public pin). + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "withheld blob is sealed as an encrypted recovery copy, not pinned in the clear" + ); + } + + /// SCENARIO 8 (no-coalesce happy path). A single push with no coalesced follower + /// runs exactly one pass, pins its object, and releases the key. No requeue. + #[sqlx::test] + async fn u3_no_coalesce_single_pass_pins_and_releases(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-happy"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // No second try_begin: the repo is never marked dirty. + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push admits"); + assert_eq!( + state.encrypt_inflight.dirty(&repo.id), + Some(false), + "clean, no coalesce" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj1.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "the single push's object is pinned" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the key is released after one pass" + ); + } + } } From 86fc2763a3ee242471b54189c79a288f91cf9890 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 20 Jul 2026 19:17:12 -0500 Subject: [PATCH 10/77] fix(node): wire GITLAWB_IPFS_MAX_REPOS_WALKED to the legacy-probe budget The knob was parsed and documented as the per-request /ipfs fan-out cap but never read in production: AppState seeded the legacy-probe budget from a fixed 256 constant, so setting the knob did nothing and an operator could still incur 256 acquire/probe operations per request. Seed ipfs_max_legacy_probes from the knob (default raised to 256 to preserve shipped behavior); leave the history-walk ceiling on its MAX_PIN_SOURCES+1 constant so a provenanced request is never truncated into a false 503. Updates the README and .env.example to the shipped behavior and adds cap-honored and plumbing tests. --- .env.example | 7 +-- README.md | 2 +- crates/gitlawb-node/src/config.rs | 57 +++++++++++++++++++++---- crates/gitlawb-node/src/main.rs | 5 ++- crates/gitlawb-node/src/state.rs | 12 ++++++ crates/gitlawb-node/src/test_support.rs | 49 +++++++++++++++++++++ 6 files changed, 118 insertions(+), 14 deletions(-) diff --git a/.env.example b/.env.example index 149ea8b1..1c390a1c 100644 --- a/.env.example +++ b/.env.example @@ -168,9 +168,10 @@ GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32 # per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map). # Default 4. GITLAWB_IPFS_WALK_PER_SOURCE=4 -# Max repos walked per single /ipfs request, so one request cannot serialize a -# full-history walk over every repo carrying the CID. Default 64. -GITLAWB_IPFS_MAX_REPOS_WALKED=64 +# Max legacy (NULL-provenance) repos probed per single /ipfs request, bounding the +# scan-fallback fan-out (git cat-file per candidate repo) for an anonymous caller. A +# truncated scan sheds a retryable 503, never a false 404. Default 256. +GITLAWB_IPFS_MAX_REPOS_WALKED=256 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. GITLAWB_IPFS_RATE_LIMIT=600 diff --git a/README.md b/README.md index e9765876..a7e4425b 100644 --- a/README.md +++ b/README.md @@ -347,7 +347,7 @@ Important node settings: | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | -| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max repos walked per `/ipfs/{cid}` request, bounding one request's fan-out. Default 64. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max legacy (NULL-provenance) repos probed per `/ipfs/{cid}` request, bounding the scan-fallback fan-out. A truncated scan returns a retryable 503, not a false 404. Default 256. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index b3667024..a9b76067 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -372,17 +372,20 @@ pub struct Config { )] pub ipfs_walk_per_source: usize, - /// Upper bound on the number of candidate repos a single `/ipfs/{cid}` request - /// will walk before giving up (returning the opaque 404). The handler already - /// short-circuits the moment it serves the object, but a CID that is present in - /// (or path-gated out of) many repos could otherwise serialize one full-history - /// walk per repo inside a single held admission slot. Capping the count bounds - /// the worst-case work one request can pin its slot with. Must be between 1 and - /// 1_048_576. Default: 64. + /// Per-request ceiling on the number of legacy (NULL-provenance) repos the + /// `/ipfs/{cid}` resolver's scan fallback will PROBE (`acquire` + `git cat-file + /// -t`) before giving up. The provenance path targets one repo; the legacy scan, + /// absent this bound, fans one anonymous request out to O(repos) subprocess spawns + /// and cold-cache fetches for a CID enumerable from the public pins index (#173, + /// INV-10). A truncated scan surfaces as a retryable 503, never a false 404. Wired + /// into `AppState::ipfs_max_legacy_probes` at construction; the history-walk ceiling + /// stays constant (`MAX_HISTORY_WALKS_PER_REQUEST`) and is NOT governed by this knob + /// (a smaller value would falsely 503 a provenanced request). Must be between 1 and + /// 1_048_576. Default: 256. #[arg( long, env = "GITLAWB_IPFS_MAX_REPOS_WALKED", - default_value_t = 64, + default_value_t = crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST as usize, value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) )] pub ipfs_max_repos_walked: usize, @@ -534,7 +537,7 @@ mod tests { fn ipfs_max_repos_walked_defaults_and_rejects_out_of_range() { assert_eq!( Config::parse_from(["gitlawb-node"]).ipfs_max_repos_walked, - 64 + 256 ); assert_eq!( Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "8"]) @@ -548,6 +551,42 @@ mod tests { ); } + /// The `GITLAWB_IPFS_MAX_REPOS_WALKED` knob must actually reach the legacy-probe + /// budget it advertises (R5, KTD5): production seeds `ipfs_max_legacy_probes` from + /// this helper, so the knob is a no-op unless the helper reflects it. RED while the + /// helper returns the hardcoded `MAX_LEGACY_PROBES_PER_REQUEST` (256 regardless of + /// the knob), GREEN once it reads the knob. + #[test] + fn ipfs_max_repos_walked_wires_the_legacy_probe_budget() { + use crate::state::AppState; + // Knob set to 1 → a one-probe legacy budget. + let one = Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1"]); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&one), + 1, + "the knob must control the legacy-probe budget, not be ignored" + ); + // Unset knob preserves the shipped 256-probe behaviour. + let default = Config::parse_from(["gitlawb-node"]); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&default), + 256, + "the default knob keeps the shipped 256-probe budget" + ); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&default), + crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + "the default budget equals the constant it replaced" + ); + // Ceiling guard: the knob never governs the history-walk ceiling, which must + // stay at MAX_PIN_SOURCES + 1 or a provenanced full source set false-503s. + assert!( + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST + >= crate::db::MAX_PIN_SOURCES as u32 + 1, + "the history-walk ceiling is independent of the repos-walked knob" + ); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 37190c76..0af288c2 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -375,7 +375,10 @@ async fn main() -> Result<()> { create_ip_rate_limiter, push_rate_limiter, ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, - ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + // The legacy-probe budget is operator-tunable via GITLAWB_IPFS_MAX_REPOS_WALKED + // (R5); the history-walk ceiling above stays constant (a smaller value false-503s + // a provenanced request). Default 256 preserves the shipped behaviour. + ipfs_max_legacy_probes: AppState::ipfs_legacy_probe_budget(&config), ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust, sync_trigger_rate_limiter, diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 61bbec6a..bc57b2d9 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -242,6 +242,18 @@ impl AppState { pub fn is_shutting_down(&self) -> bool { *self.shutdown_tx.borrow() } + + /// Legacy-probe budget wired from the `GITLAWB_IPFS_MAX_REPOS_WALKED` operator + /// knob (R5, KTD5). The knob seeds `ipfs_max_legacy_probes` at construction so it + /// controls the per-request legacy (NULL-provenance) probe fan-out it advertises. + /// It deliberately does NOT feed `ipfs_max_history_walks`: that ceiling must stay + /// at `MAX_HISTORY_WALKS_PER_REQUEST` (`MAX_PIN_SOURCES + 1`) or a provenanced + /// request with a full source set is truncated into a false 503, and the knob's + /// range starts at 1. The knob is `usize`, the field `u32`; the range cap + /// (1_048_576) keeps the cast lossless. + pub(crate) fn ipfs_legacy_probe_budget(config: &crate::config::Config) -> u32 { + config.ipfs_max_repos_walked as u32 + } } /// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 20e325ac..30f63d83 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -3674,6 +3674,55 @@ mod tests { ); } + /// T2b (R5, KTD5): the `GITLAWB_IPFS_MAX_REPOS_WALKED` knob drives the legacy-probe + /// budget end to end. With the knob at 1 (fed through the same production helper the + /// state seeding uses) and two candidate repos that miss, the first repo spends the + /// single probe and the second is skipped at the cap → truncated → 503. If the knob + /// budget were not honoured (unbounded), both would probe, both miss, and the request + /// would be a definitive 404. Proves the wired knob=1 → exactly one probe path. + #[sqlx::test] + async fn ipfs_cid_repos_walked_knob_caps_legacy_probes(pool: PgPool) { + use clap::Parser; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Seed the legacy-probe budget the way production does: from the operator knob. + let cfg = + crate::config::Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1"]); + state.ipfs_max_legacy_probes = AppState::ipfs_legacy_probe_budget(&cfg); + assert_eq!(state.ipfs_max_legacy_probes, 1, "knob=1 → one-probe budget"); + // The knob must not touch the history-walk ceiling (must stay MAX_PIN_SOURCES + 1). + assert_eq!( + state.ipfs_max_history_walks, + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + "the repos-walked knob leaves the history-walk ceiling untouched" + ); + + let _fx = seed_cid_repos(&slug, &short, &["k0", "k1"]); + for n in ["k0", "k1"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo: the cap, not a hit, decides. + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-knob").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "knob=1 caps the scan at one probe → incomplete search → retryable 503" + ); + } + /// T3 (F2): a walk-cap truncation must not false-404. Walk ceiling shrunk to 1; /// two public repos each carry a path-scoped rule over the object and deny anon. /// The 1st spends the single walk (deny), the 2nd is skipped at the cap — the From 20c338eca5ac36b38a9b4614b61589f678a7bb81 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 20 Jul 2026 19:41:57 -0500 Subject: [PATCH 11/77] fix(node): split /ipfs work accounting off the route rate limiter The route middleware charged ipfs_rate_limiter once per request while the legacy scan and provenance walk charged the same bucket per probe/walk, so a one-probe request cost two tokens and GITLAWB_IPFS_RATE_LIMIT=1 admitted the request at the route then 429'd it at the pre-scan peek. Add a bounded ipfs_work_rate_limiter, move the in-scan, per-walk, and pre-scan charges onto it, and leave the route limiter as the pure once-per-request brake. The work-budget capacity derives from the route limit with a floor of one full legacy-probe budget per window (no new operator knob), so a default-config deep search never self-throttles. Adds it to the periodic sweep, updates all three AppState construction sites, and rewrites the contradictory limiter comments. --- crates/gitlawb-node/src/api/ipfs.rs | 36 +++-- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/config.rs | 83 ++++++++++- crates/gitlawb-node/src/main.rs | 9 ++ crates/gitlawb-node/src/state.rs | 46 +++++- crates/gitlawb-node/src/test_support.rs | 182 ++++++++++++++++++++++-- 6 files changed, 322 insertions(+), 35 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 016e130d..c5fa35bc 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -38,11 +38,12 @@ use crate::state::AppState; use crate::visibility::{visibility_check, Decision}; /// Hard ceiling on the number of full-history reachability walks a single -/// `GET /ipfs/{cid}` request may spawn. The per-request `ipfs_rate_limiter` -/// check brakes *repeat* requests, but within one request the object can exist -/// under path-scoped rules in many repos, and each distinct repo pays its own -/// `spawn_blocking` walk (the memo only dedups the same repo). Without a ceiling -/// a single request fans out to O(repos) walks for one rate-limiter token — an +/// `GET /ipfs/{cid}` request may spawn. The route brake (`ipfs_rate_limiter`, charged +/// once per request by the middleware) caps request RATE, and the per-walk charge on +/// the separate `ipfs_work_rate_limiter` bounds the walk work across requests, but +/// within ONE request the object can exist under path-scoped rules in many repos, and +/// each distinct repo pays its own `spawn_blocking` walk (the memo only dedups the same +/// repo). Without a ceiling a single request fans out to O(repos) walks — an /// amplification sink (INV-10). Once this many walks have run, no further walk is /// spawned for the rest of the request: any remaining candidate that still needs /// a walk is skipped (and, with nothing else readable, the request falls through @@ -335,14 +336,18 @@ pub async fn get_by_cid( .await .map_err(AppError::Internal)?; if needs_scan { - // F3 (#173, INV-10/INV-15): peek the per-IP limiter WITHOUT consuming a token - // so an already-throttled source is shed BEFORE the O(repos) preload; the - // consuming per-probe charge inside gate_and_serve is left UNCHANGED (it is - // load-bearing for the across-request bound), so this adds no double-charge. + // F3 (#173, INV-10/INV-15): peek the per-IP WORK-budget limiter WITHOUT + // consuming a token so an already-throttled source is shed BEFORE the + // O(repos) preload; the consuming per-probe charge inside gate_and_serve is + // left UNCHANGED (it is load-bearing for the across-request bound), so this + // adds no double-charge. This peeks `ipfs_work_rate_limiter`, the SAME bucket + // the per-probe charge below debits — NOT the route limiter (`ipfs_rate_limiter`, + // charged once per request by the middleware): peeking the route bucket here + // would re-shed a request the route already admitted (R6, U5). if let Some(key) = crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) { - if state.ipfs_rate_limiter.is_throttled(&key).await { + if state.ipfs_work_rate_limiter.is_throttled(&key).await { throttled = true; continue; } @@ -536,9 +541,10 @@ async fn gate_and_serve( // reset each request, leaving a NULL-provenance CID open to unbounded ACROSS- // request amplification: N requests spending N x budget cold `acquire` calls // against Tigris with zero limiter contact (#173, F3, jatmn). Charging the first - // probe makes those requests accumulate against the per-IP `ipfs_rate_limiter`, - // closing that path. The per-request cap below stays as the second bound (a - // single request's ceiling). A spent quota is the same non-fatal Throttled as the + // probe makes those requests accumulate against the per-IP `ipfs_work_rate_limiter` + // (the resolver's WORK bucket, separate from the once-per-request route brake + // `ipfs_rate_limiter` — R6, U5), closing that path. The per-request cap below stays + // as the second bound (a single request's ceiling). A spent quota is the same non-fatal Throttled as the // walk brake: keep scanning for a walk-free copy, and only a wholly-unservable // request becomes the 429. No resolvable key (a test oneshot with no peer/header) // skips the brake, as the walk brake does. The provenance path targets one repo @@ -553,7 +559,7 @@ async fn gate_and_serve( if let Some(key) = crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) { - if !state.ipfs_rate_limiter.check(&key).await { + if !state.ipfs_work_rate_limiter.check(&key).await { return GateOutcome::Throttled; } } @@ -658,7 +664,7 @@ async fn gate_and_serve( if let Some(key) = crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) { - if !state.ipfs_rate_limiter.check(&key).await { + if !state.ipfs_work_rate_limiter.check(&key).await { return GateOutcome::Throttled; } } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index c03eb2e5..96e6f5ea 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -517,6 +517,7 @@ mod tests { create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index a9b76067..5480beda 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -396,6 +396,11 @@ pub struct Config { /// concurrency cap above (a rate limit bounds request *rate*, the semaphore /// bounds concurrent slow holds — different axes). Keyed on the resolved client /// IP via `GITLAWB_TRUSTED_PROXY`. `0` disables. Default: 600. + /// + /// This is the pure once-per-request ROUTE brake. The resolver's internal + /// per-probe/per-walk WORK budget is a SEPARATE bucket whose capacity is DERIVED + /// from this value (`AppState::ipfs_work_budget`), not a knob of its own; `0` here + /// disables that derived bucket too. #[arg(long, env = "GITLAWB_IPFS_RATE_LIMIT", default_value_t = 600)] pub ipfs_rate_limit: usize, } @@ -581,12 +586,86 @@ mod tests { // Ceiling guard: the knob never governs the history-walk ceiling, which must // stay at MAX_PIN_SOURCES + 1 or a provenanced full source set false-503s. assert!( - crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST - >= crate::db::MAX_PIN_SOURCES as u32 + 1, + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST > crate::db::MAX_PIN_SOURCES as u32, "the history-walk ceiling is independent of the repos-walked knob" ); } + /// The `/ipfs` work-budget capacity is DERIVED from the route limit (R6, KTD6), with + /// a hard floor of one full legacy search per window (the effective + /// `ipfs_max_legacy_probes`). This guards the derived default so a single + /// default-config deep search never self-throttles mid-scan and recreates the F6 + /// admit-then-429 for a legitimate caller. A `RateLimiter` sized to the derived + /// budget must admit the whole probe budget back to back. + #[test] + fn ipfs_work_budget_derives_from_route_limit_and_clears_the_probe_floor() { + use crate::state::AppState; + + // Default config: derived work budget = max(route 600, probe budget 256) = 600, + // comfortably above the 256-probe floor. + let default = Config::parse_from(["gitlawb-node"]); + let budget = AppState::ipfs_work_budget(&default); + assert_eq!(budget, 600, "default derives max(route 600, probe 256)"); + assert!( + budget >= AppState::ipfs_legacy_probe_budget(&default) as usize, + "the work budget must clear one full legacy search per window" + ); + + // Tight route limit (1): the floor lifts the work budget to the probe budget + // (256), NOT down to 1 — a single deep search still completes its full scan. + let tight = Config::parse_from(["gitlawb-node", "--ipfs-rate-limit", "1"]); + assert_eq!( + AppState::ipfs_work_budget(&tight), + 256, + "a tight route limit is floored at the 256-probe budget, not clamped to 1" + ); + + // Raised probe budget lifts the floor with it (the work budget tracks the + // effective probe budget, not the constant). + let raised = Config::parse_from([ + "gitlawb-node", + "--ipfs-rate-limit", + "10", + "--ipfs-max-repos-walked", + "1000", + ]); + assert_eq!( + AppState::ipfs_work_budget(&raised), + 1000, + "the floor tracks the operator-raised legacy-probe budget" + ); + + // 0 route limit disables the derived bucket too (a 0-capacity limiter admits all). + let disabled = Config::parse_from(["gitlawb-node", "--ipfs-rate-limit", "0"]); + assert_eq!( + AppState::ipfs_work_budget(&disabled), + 0, + "route limit 0 disables the derived work bucket alongside the route brake" + ); + + // Behavioral floor: a limiter sized to the derived (tight-route) budget admits + // the whole probe budget back to back for one source, then sheds the next. + let budget = AppState::ipfs_work_budget(&tight); + let limiter = + crate::rate_limit::RateLimiter::new(budget, std::time::Duration::from_secs(3600)); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async { + for i in 0..budget { + assert!( + limiter.check("1.2.3.4").await, + "probe {i} of one full default-config scan must be admitted (no mid-scan throttle)" + ); + } + assert!( + !limiter.check("1.2.3.4").await, + "the probe past the derived budget is shed" + ); + }); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 0af288c2..1ed45929 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -437,6 +437,15 @@ async fn main() -> Result<()> { std::time::Duration::from_secs(3600), 200_000, ), + // Separate WORK-budget bucket for the resolver's per-probe/per-walk charges (R6). + // Its capacity is DERIVED from the route limit (no new knob) and floored at the + // legacy-probe budget, so one full default-config legacy scan never self-throttles + // mid-request while the route brake above stays the pure once-per-request cap. + ipfs_work_rate_limiter: rate_limit::RateLimiter::new_bounded( + AppState::ipfs_work_budget(&config), + std::time::Duration::from_secs(3600), + 200_000, + ), git_bin: "git".to_string(), }; if config.ipfs_rate_limit == 0 { diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index bc57b2d9..51618b78 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -65,13 +65,29 @@ pub struct AppState { /// brake a push flood from a DID farm (one throwaway DID per repo), so the /// push path throttles on the resolved client IP instead. pub push_rate_limiter: RateLimiter, - /// Per-client-IP rate limiter for the `GET /ipfs/{cid}` full-history walk. - /// The route is anonymous and a valid tree CID (exposed by the public pins - /// index) makes each repeat request pay a fresh allowed-set walk (rev-list + - /// ls-tree per commit), memoized only per request — unbounded amplification - /// (INV-10). Braking the walk on the non-farmable source IP caps that cost - /// without touching cheap non-walk fetches. Keyed by `push_limiter_trust`. + /// Per-client-IP ROUTE brake for `GET /ipfs/{cid}`: charged ONCE per request by the + /// `rate_limit_by_ip` middleware (server.rs), never inside the handler. It bounds + /// request RATE (the "requests per hour" contract of `GITLAWB_IPFS_RATE_LIMIT`) on + /// the non-farmable source IP, so an anonymous flood of the public route is capped. + /// The per-probe/per-walk WORK accounting the resolver does WITHIN a request draws + /// from the SEPARATE `ipfs_work_rate_limiter` below — the two cannot share one bucket + /// or a single request that spends a route token and then its own probe token off the + /// same bucket is admitted at the route and falsely shed mid-request (#173 round-10, + /// R6). Keyed by `push_limiter_trust`. pub ipfs_rate_limiter: RateLimiter, + /// Per-client-IP WORK-budget limiter for the `GET /ipfs/{cid}` resolver's internal + /// fan-out: charged per legacy (NULL-provenance) PROBE (`acquire` + `cat-file`) and + /// per provenance-path WALK, and peeked non-consuming before the O(repos) legacy + /// preload. A legacy CID from the public pins index otherwise lets one request drive + /// O(repos) subprocess spawns and cold Tigris fetches, and repeat requests amplify + /// that across requests with zero limiter contact (INV-10, F3). Charging the work to + /// the non-farmable source IP bounds it. A bucket DISTINCT from the route brake above: + /// one request legitimately spends many work tokens (a full legacy scan is up to + /// `ipfs_max_legacy_probes` probes), so it must not double as the once-per-request + /// route bucket. Capacity is DERIVED from the route limit (`AppState::ipfs_work_budget`, + /// no separate operator knob), floored at the legacy-probe budget so a single default- + /// config deep search never self-throttles mid-scan. Keyed by `push_limiter_trust`. + pub ipfs_work_rate_limiter: RateLimiter, /// Per-request ceiling on full-history reachability walks the CID resolver /// may spawn (default `api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST`). A field, /// not a bare const, so tests can shrink it to exercise the cap cheaply; @@ -218,6 +234,7 @@ impl AppState { self.create_ip_rate_limiter.cleanup().await; self.push_rate_limiter.cleanup().await; self.ipfs_rate_limiter.cleanup().await; + self.ipfs_work_rate_limiter.cleanup().await; self.sync_trigger_rate_limiter.cleanup().await; self.peer_write_rate_limiter.cleanup().await; } @@ -254,6 +271,23 @@ impl AppState { pub(crate) fn ipfs_legacy_probe_budget(config: &crate::config::Config) -> u32 { config.ipfs_max_repos_walked as u32 } + + /// Work-budget capacity for [`ipfs_work_rate_limiter`](Self#structfield.ipfs_work_rate_limiter) + /// (R6, KTD6), DERIVED from the route limit rather than a new operator knob. The route + /// limiter (`ipfs_rate_limiter`) charges once per request; this separate bucket absorbs + /// the resolver's per-probe/per-walk work charges so both the route "requests per hour" + /// contract and the amplification bound hold. Floor: at least one full legacy search per + /// window — the effective `ipfs_max_legacy_probes` (the `GITLAWB_IPFS_MAX_REPOS_WALKED` + /// knob, U4) — so a single default-config deep search cannot self-throttle mid-scan and + /// recreate the F6 admit-then-429 for a legitimate caller. `GITLAWB_IPFS_RATE_LIMIT=0` + /// disables the route brake and this derived bucket alike (a 0-capacity limiter admits + /// everything). + pub(crate) fn ipfs_work_budget(config: &crate::config::Config) -> usize { + if config.ipfs_rate_limit == 0 { + return 0; + } + config.ipfs_rate_limit.max(config.ipfs_max_repos_walked) + } } /// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 30f63d83..01363abf 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -79,6 +79,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, @@ -3285,7 +3286,8 @@ mod tests { let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let fx = seed_cid_repos(&slug, &short, &["provthrottle"]); @@ -3455,7 +3457,8 @@ mod tests { let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let fx = seed_cid_repos(&slug, &short, &["fanout"]); @@ -3498,7 +3501,7 @@ mod tests { /// fresh batch of `acquire` + `cat-file` probes every request with zero limiter /// contact, unbounded anonymous amplification against Tigris. Charging every /// legacy probe from the first one makes those probes accumulate against the - /// per-IP `ipfs_rate_limiter` ACROSS requests. Four repos, none holding the CID, + /// per-IP `ipfs_work_rate_limiter` ACROSS requests. Four repos, none holding the CID, /// so a full scan probes all four; the per-IP budget is sized to exactly ONE such /// scan (4 tokens). req1 (a genuine absence) fully scans and 404s, spending the /// budget; req2 from the SAME IP is shed at the first probe → 429 (it never @@ -3515,7 +3518,8 @@ mod tests { let mut state = test_state(pool).await; // Budget = one full scan of the four seeded repos. A repeat scan from the same // IP then finds it spent. Keyed on XFF so `oneshot` can choose the source IP. - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(4, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(4, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let names = ["a0", "a1", "a2", "a3"]; @@ -3578,7 +3582,8 @@ mod tests { let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; // Budget 1, keyed on XFF so `oneshot` can choose the source IP. - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let _fx = seed_cid_repos(&slug, &short, &["r0"]); @@ -5298,7 +5303,8 @@ mod tests { // size the per-IP budget to admit exactly one full scan (2 probes). A repeat // scan from the same IP then finds the bucket spent. Keyed on the rightmost // X-Forwarded-For hop so the test can choose a source IP under `oneshot`. - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let fx = seed_cid_repos(&slug, &short, &["walklimit"]); @@ -5504,7 +5510,8 @@ mod tests { // Budget = one full two-repo scan (2 probes), keyed on the rightmost XFF hop // so `oneshot` can choose a source IP (no socket peer). A repeat scan from the // same IP then finds the budget spent. - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; // Identical secret-blob content in both bare clones → one CID resolves to @@ -5578,10 +5585,11 @@ mod tests { } /// INV-10 amplification bound: a single `GET /ipfs/{cid}` must not fan out an - /// unbounded number of full-history walks. The per-request `ipfs_rate_limiter` - /// check only brakes REPEAT requests (it fires once per request); within one - /// request the same object can exist under path-scoped rules in many repos, - /// each paying its own walk. `MAX_HISTORY_WALKS_PER_REQUEST` caps that fan-out. + /// unbounded number of full-history walks. The route brake (`ipfs_rate_limiter`) + /// fires once per request and the per-walk `ipfs_work_rate_limiter` charge bounds + /// walk work across requests, but within ONE request the same object can exist under + /// path-scoped rules in many repos, each paying its own walk. + /// `MAX_HISTORY_WALKS_PER_REQUEST` caps that fan-out. /// /// Load-bearing witness (#173, F4): a readable public copy (no path rule → /// served via the no-walk path, exactly like @@ -5784,7 +5792,8 @@ mod tests { let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let fx = seed_cid_repos(&slug, &short, &["w0", "w1"]); @@ -5851,6 +5860,155 @@ mod tests { ); } + /// U5 (R6, KTD6), the observed defect: the `/ipfs` route rate limit and the + /// resolver's per-probe WORK budget are SEPARATE buckets, so a single request with + /// one probe COMPLETES even at route limit = 1. Through the production router the + /// `rate_limit_by_ip` middleware charges `ipfs_rate_limiter` once (its 1-slot bucket + /// is now full); the handler's legacy pre-scan peek and per-probe charge then draw + /// from `ipfs_work_rate_limiter`, a different bucket, so the walk-free public copy + /// still serves 200. RED before the split (both charges on `ipfs_rate_limiter`): the + /// middleware fills the one slot, the pre-scan peek reads it throttled, nothing is + /// servable → 429 on the FIRST request. Trust None so the middleware and the handler + /// resolve the same `ConnectInfo` peer IP. + #[sqlx::test] + async fn ipfs_route_limit_1_still_serves_one_probe(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(600, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Public, no-rule legacy pin (NULL provenance) → the resolver takes the scan + // fallback and serves walk-free (exactly one probe). + let fx = seed_cid_repos(&slug, &short, &["routeone"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("routeone.git"); + let repo = seed_repo(&owner_did, "routeone"); + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + let router = crate::server::build_router(state); + let peer: std::net::SocketAddr = "203.0.113.7:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut() + .insert(axum::extract::ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::OK, + "a single /ipfs request with one probe must serve even at route limit = 1 \ + (the route brake and the resolver's work budget are separate buckets)" + ); + } + + /// U5 (R6): the two buckets are independent — the WORK budget can be exhausted + /// (429) WITHOUT draining the ROUTE bucket. Through the production router, route + /// generous (5) but work tight (1): one request drives two legacy probes, so the + /// second probe finds the work bucket spent → 429 (the route middleware admitted it). + /// The route bucket, charged once by the middleware, still has room afterward — the + /// work charges never touched it, so it admits four more direct checks. + #[sqlx::test] + async fn ipfs_work_exhaustion_leaves_route_bucket_intact(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(5, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // A legacy pin absent from every repo so the scan probes both seeded repos: two + // probes, work budget 1 → the second probe is shed → 429. + let names = ["we0", "we1"]; + let _fx = seed_cid_repos(&slug, &short, &names); + for n in names { + state + .db + .create_repo(&seed_repo(&owner_did, n)) + .await + .expect("seed repo"); + } + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"work-exhaustion").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("legacy pin"); + + let route_bucket = state.ipfs_rate_limiter.clone(); + let peer_ip = "203.0.113.8"; + let peer: std::net::SocketAddr = format!("{peer_ip}:5000").parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut() + .insert(axum::extract::ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "a request whose probes exceed the work budget is shed 429 (work bucket), \ + not blocked at the route (route bucket generous)" + ); + // The route bucket recorded only the single request the middleware charged; the + // work charges did not drain it. Sized 5, one used by the request → four left. + for i in 0..4 { + assert!( + route_bucket.check(peer_ip).await, + "route check {i} must still admit — work charges never drained the route bucket" + ); + } + } + + /// U5 (R6): the periodic cleanup task sweeps the NEW work-budget limiter too, not + /// only the route limiter and its siblings. Mirrors + /// `sweep_rate_limiters_includes_ipfs_limiter`: drive `sweep_rate_limiters` and + /// assert the work limiter's expired entry is evicted. Dropping the + /// `ipfs_work_rate_limiter.cleanup()` call from that method leaves the entry in place + /// (`tracked_keys` stays 1): the RED proof the sweep covers it. + #[sqlx::test] + async fn sweep_rate_limiters_includes_ipfs_work_limiter(pool: PgPool) { + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(5, Duration::from_millis(50)); + + assert!( + state.ipfs_work_rate_limiter.check("1.2.3.4").await, + "record a hit on the work limiter" + ); + assert_eq!( + state.ipfs_work_rate_limiter.tracked_keys().await, + 1, + "the source-IP key is tracked before the sweep" + ); + + tokio::time::sleep(Duration::from_millis(60)).await; + state.sweep_rate_limiters().await; + + assert_eq!( + state.ipfs_work_rate_limiter.tracked_keys().await, + 0, + "the periodic sweep must evict the work limiter's expired entries" + ); + } + // --------------------------------------------------------------------------- // Issue #120 — repo-scoped read surfaces visibility gate // --------------------------------------------------------------------------- From c04c4a5da480fd81a131eb9389f82873df2f0c2b Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 20 Jul 2026 19:50:23 -0500 Subject: [PATCH 12/77] fix(node): retry pin-source and pinned-CID recording on transient errors The three warn-only record sites in the detached post-push task dropped a transient DB failure silently, leaving a permanently incomplete pin-source set that makes the resolver 404 a valid public copy. Wrap them in a bounded retry (inserts are idempotent via ON CONFLICT DO NOTHING); on exhaustion the warn still fires so behavior degrades to today's. Documents the residual crash/outage window that only a reconciliation sweep retires. --- crates/gitlawb-node/src/ipfs_pin.rs | 135 +++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index e70d8f7b..1a37005d 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -9,6 +9,44 @@ use anyhow::Result; use gitlawb_core::cid::Cid; +use std::time::Duration; + +/// Attempts (including the first) for a transient DB-record retry. +const PIN_RECORD_ATTEMPTS: u32 = 3; +/// Backoff between DB-record retry attempts. +const PIN_RECORD_BACKOFF: Duration = Duration::from_millis(50); + +/// Run an idempotent DB-record operation with a bounded retry so a sub-second +/// transient error does not silently leave the pin-source set permanently +/// incomplete. The resolver treats a nonempty below-cap source set as complete, +/// so a dropped `record_pin_source`/`record_pinned_cid` makes `GET /ipfs/{cid}` +/// 404 a valid public copy. Every wrapped insert is idempotent (`ON CONFLICT DO +/// NOTHING` / provenance-preserving upsert), so re-running is safe. On exhausted +/// attempts the last error is returned and the caller keeps its warn — behavior +/// degrades to the pre-retry state, not worse. Process death mid-retry or a DB +/// outage outlasting the backoff horizon leaves the same residual hole (no +/// persisted marker to reconcile from at startup), retired only by a future +/// reconciliation sweep. Runs inside the already-detached post-push task, so the +/// backoff adds no push latency. +async fn retry_db_record(mut op: F) -> Result<()> +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let mut attempt = 1; + loop { + match op().await { + Ok(()) => return Ok(()), + Err(e) => { + if attempt >= PIN_RECORD_ATTEMPTS { + return Err(e); + } + tokio::time::sleep(PIN_RECORD_BACKOFF).await; + attempt += 1; + } + } + } +} /// Pin a single git object to the local IPFS/Kubo node. /// @@ -134,7 +172,7 @@ pub async fn pin_new_objects( // and without it `GET /ipfs/{cid}` only ever knows the first pinner, so a // shared object first pinned from a private/quarantined repo 404s even // when this repo would serve it. Bounded per object (MAX_PIN_SOURCES). - if let Err(e) = db.record_pin_source(&sha, repo_id).await { + if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); } continue; @@ -167,12 +205,14 @@ pub async fn pin_new_objects( // verifies them against the requested CID, so the raw CID is the correct // key. Mirrors the pinata twin, which already records the raw CID. let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); - if let Err(e) = db.record_pinned_cid(&sha, &raw_cid, Some(repo_id)).await { + if let Err(e) = + retry_db_record(|| db.record_pinned_cid(&sha, &raw_cid, Some(repo_id))).await + { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); } // F1 (#173 round 8): also record the first pinner in pin_repo_sources so // every source (first and subsequent) is tried uniformly by the resolver. - if let Err(e) = db.record_pin_source(&sha, repo_id).await { + if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); } // Return the provider Hash (not the resolver key), mirroring the pinata @@ -190,3 +230,92 @@ pub async fn pin_new_objects( pinned } + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + // The retry helper is the load-bearing unit: it converts a sub-second + // transient DB error at the three warn-only record sites into a landed row, + // instead of a permanently incomplete pin-source set. These drive the helper + // directly against a controlled closure (the record sites take a concrete + // `&Db` over a `PgPool`, so a failing-first wrapper cannot slot in without + // changing signatures — see U6 seam note). + + #[tokio::test] + async fn retry_lands_after_transient_failures() { + let calls = Cell::new(0u32); + let result = retry_db_record(|| { + let n = calls.get() + 1; + calls.set(n); + async move { + if n < PIN_RECORD_ATTEMPTS { + Err(anyhow::anyhow!("transient failure on attempt {n}")) + } else { + Ok(()) + } + } + }) + .await; + + assert!( + result.is_ok(), + "retry lands the row after transient failures" + ); + assert_eq!( + calls.get(), + PIN_RECORD_ATTEMPTS, + "op is retried until it succeeds" + ); + } + + #[tokio::test] + async fn retry_returns_last_err_after_exhaustion() { + let calls = Cell::new(0u32); + let result = retry_db_record(|| { + let n = calls.get() + 1; + calls.set(n); + async move { Err::<(), _>(anyhow::anyhow!("attempt {n} failed")) } + }) + .await; + + let err = result.expect_err("all attempts fail so the last error surfaces"); + assert_eq!( + calls.get(), + PIN_RECORD_ATTEMPTS, + "attempts are bounded to the cap" + ); + assert_eq!( + err.to_string(), + "attempt 3 failed", + "the LAST error is returned, not the first" + ); + } + + // Happy path against a real DB: a single-attempt success lands the row, and a + // redundant call is idempotent (`ON CONFLICT DO NOTHING`), so the source set + // holds exactly one row. + #[sqlx::test] + async fn retry_records_pin_source_once(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let sha = "a".repeat(64); + let repo_id = "repo-retry-1"; + + retry_db_record(|| db.record_pin_source(&sha, repo_id)) + .await + .expect("happy-path record succeeds in one attempt"); + retry_db_record(|| db.record_pin_source(&sha, repo_id)) + .await + .expect("a redundant record is idempotent"); + + let sources = db.pin_sources_for_oid(&sha).await.unwrap(); + assert_eq!( + sources, + vec![repo_id.to_string()], + "exactly one source row lands under ON CONFLICT DO NOTHING" + ); + } +} From 15bdc5e64aed090b2eee39ae5a561fa01a045c46 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 20 Jul 2026 20:11:42 -0500 Subject: [PATCH 13/77] fix(node): opportunistically repair legacy provider-CID pins Before this PR the pin path stored the provider CID (Kubo dag-pb / Pinata) in pinned_cids.cid, which the new resolver recomputes-and-404s, while /api/v1/ipfs/pins still advertises the stale value. Repair a legacy row in the already-pinned skip branch: gate on a cheap stored-CID codec check (only a non-raw-codec CID reads bytes, so raw rows keep the DB-only skip cost), recompute the raw CID, and rewrite it while stashing the old value in a new v14 legacy_provider_cid column (distinct from pinata_cid, which gates the Pinata pin-skip). Rows whose bytes are gone stay withheld. Documents that the deferred one-shot sweep, not this opportunistic path, fully retires the window. --- README.md | 2 + crates/gitlawb-core/src/cid.rs | 45 ++++ crates/gitlawb-node/src/db/mod.rs | 57 ++++ crates/gitlawb-node/src/ipfs_pin.rs | 76 ++++++ crates/gitlawb-node/src/test_support.rs | 329 ++++++++++++++++++++++++ 5 files changed, 509 insertions(+) diff --git a/README.md b/README.md index a7e4425b..80f71cb7 100644 --- a/README.md +++ b/README.md @@ -355,6 +355,8 @@ Important node settings: Production note: change the default Postgres password before exposing a node publicly. +Legacy-pin window: releases before the CID-resolver work stored the provider CID (Kubo dag-pb / Pinata) as a pinned object's resolver key. The `/ipfs/{cid}` resolver now recomputes the raw-content CID from the object bytes and refuses to serve a key that does not match, so `GET /api/v1/ipfs/pins` can still advertise an unrepaired legacy CID that 404s. Such a row is repaired opportunistically the next time a push carries the object again (its key is rewritten to the raw CID, the old value kept in `legacy_provider_cid`), but git negotiation omits objects the node already has, so most legacy rows never re-enter a push delta. A deferred one-shot startup sweep, not this opportunistic path, is what fully retires the advertise-then-404 window. Rows whose object bytes are gone stay withheld. + --- ## Optional node staking diff --git a/crates/gitlawb-core/src/cid.rs b/crates/gitlawb-core/src/cid.rs index b7993cc4..2071d478 100644 --- a/crates/gitlawb-core/src/cid.rs +++ b/crates/gitlawb-core/src/cid.rs @@ -64,6 +64,19 @@ impl Cid { } } +/// True when `s` parses as a CIDv1 with the raw codec — the exact shape +/// [`Cid::from_git_object_bytes`] produces and the `/ipfs` resolver looks up. +/// A legacy provider CID (Kubo dag-pb, Pinata CIDv0) parses to a different +/// version or codec and returns `false`, marking it an opportunistic-repair +/// candidate. Decidable from the string alone (no object bytes), so the pin path +/// can gate the byte-read/recompute cost on it and leave non-legacy rows at the +/// existing DB-only skip cost. An unparseable string is non-canonical (`false`). +pub fn is_raw_cidv1(s: &str) -> bool { + s.parse::>() + .map(|c| c.version() == cid::Version::V1 && c.codec() == RAW) + .unwrap_or(false) +} + impl fmt::Display for Cid { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.0) @@ -177,6 +190,38 @@ mod tests { assert!(result.is_err()); } + #[test] + fn is_raw_cidv1_classifies_codec_from_string() { + // The canonical resolver key: CIDv1 + raw codec → not a repair candidate. + let raw = Cid::from_git_object_bytes(b"blob 5\0hello"); + assert!( + is_raw_cidv1(raw.as_str()), + "from_git_object_bytes output is CIDv1/raw" + ); + + // A CIDv0 (Pinata dag-pb legacy shape) → repair candidate. + assert!( + !is_raw_cidv1("QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"), + "a CIDv0 dag-pb value is a legacy-repair candidate" + ); + + // A CIDv1 with the dag-pb codec (the Kubo above-block-size root) over the + // same multihash → still a repair candidate (codec, not just version). + let parsed = raw.as_str().parse::>().unwrap(); + const DAG_PB: u64 = 0x70; + let dagpb = CidGeneric::<64>::new_v1(DAG_PB, *parsed.hash()).to_string(); + assert!( + !is_raw_cidv1(&dagpb), + "a CIDv1 dag-pb value is a legacy-repair candidate" + ); + + // Garbage is non-canonical. + assert!( + !is_raw_cidv1("not-a-cid"), + "an unparseable string is non-canonical" + ); + } + #[test] fn sha256_hex_of_empty_input_is_well_known() { // SHA-256("") is a fixed constant; verifies the hasher is wired correctly. diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 8d723ffd..775e7c17 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -921,6 +921,22 @@ const MIGRATIONS: &[Migration] = &[ "CREATE INDEX IF NOT EXISTS idx_pin_repo_sources_sha ON pin_repo_sources(sha256_hex)", ], }, + Migration { + version: 14, + name: "pinned_cids_legacy_provider_cid", + stmts: &[ + // R8 (#173, jatmn round 10): the opportunistic legacy provider-CID repair + // rewrites `pinned_cids.cid` from a stored PROVIDER CID (Kubo dag-pb / + // Pinata CIDv0) to the raw-content resolver key and stashes the OLD value + // here, so the rewrite is auditable and the row's legacy origin survives. + // Distinct from `pinata_cid` on purpose: `has_pinata_cid` gates the Pinata + // pin-skip, so parking a Kubo-legacy CID there would make Pinata forever + // skip re-pinning that object. NEW versioned migration (never appended to an + // applied block, INV-7) so a node past v13 actually gets the column. + // Nullable: only a repaired row sets it. + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS legacy_provider_cid TEXT", + ], + }, ]; /// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). @@ -2287,6 +2303,47 @@ impl Db { Ok(()) } + /// The resolver key currently stored for a pinned object (`pinned_cids.cid`), + /// or `None` for an unpinned oid. The opportunistic legacy-repair path reads + /// it to decide candidacy from the codec of the string alone (no object bytes) + /// before it recomputes anything. + pub async fn cid_for_oid(&self, sha256_hex: &str) -> Result> { + let row = sqlx::query("SELECT cid FROM pinned_cids WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.get::("cid"))) + } + + /// Rewrite a legacy provider-CID row to the raw-content resolver key, stashing + /// the old provider value in `legacy_provider_cid` (#173 R8, KTD8). Before this + /// branch the pin path stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in + /// `cid`; the `/ipfs` resolver recomputes the raw CID and 404s a mismatched key + /// even though `list_pinned_cids` still advertises it. The `WHERE cid = + /// $old_provider_cid` guard makes a concurrent double-repair a no-op (the second + /// writer sees the already-rewritten key and matches nothing) and never touches + /// a row keyed on a different value. Stashed in `legacy_provider_cid`, NOT + /// `pinata_cid`: the latter gates the Pinata pin-skip (`has_pinata_cid`), so a + /// Kubo-legacy CID parked there would make Pinata permanently skip the object. + pub async fn repair_legacy_provider_cid( + &self, + sha256_hex: &str, + raw_cid: &str, + old_provider_cid: &str, + ) -> Result<()> { + sqlx::query( + "UPDATE pinned_cids + SET cid = $2, legacy_provider_cid = $3 + WHERE sha256_hex = $1 AND cid = $3", + ) + .bind(sha256_hex) + .bind(raw_cid) + .bind(old_provider_cid) + .execute(&self.pool) + .await?; + Ok(()) + } + /// The repository a pinned object was recorded from (`pinned_cids.repo_id`), /// or `None` for a legacy pin (recorded before provenance existed) or an /// unpinned oid. `GET /ipfs/{cid}` uses this to gate+serve the ONE source diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 1a37005d..3cf5610c 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -48,6 +48,74 @@ where } } +/// Opportunistically repair a legacy provider-CID row on the already-pinned skip +/// path (#173 R8, KTD8). Releases before this branch stored the PROVIDER CID +/// (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`; the `/ipfs` resolver +/// recomputes the raw CID from object bytes and 404s any row whose key does not +/// match, yet `list_pinned_cids` still advertises the stored key — so a client +/// gets a CID the resolver deliberately withholds. When a re-push carries the +/// object again, rewrite the key to the raw CID and stash the old provider value +/// in `legacy_provider_cid`. +/// +/// COST GATE: candidacy is decided from the stored key's codec alone — a +/// CIDv1/raw key is already the resolver key and reads NO bytes, keeping the +/// steady-state skip cost DB-only. Only a legacy-codec row reads the object to +/// recompute. A row whose bytes are gone stays withheld (no destructive rewrite). +async fn repair_legacy_provider_cid( + repo_path: &std::path::Path, + sha: &str, + db: &crate::db::Db, +) -> Result<()> { + let stored = match db.cid_for_oid(sha).await? { + Some(c) => c, + None => return Ok(()), + }; + // Cost gate: a canonical raw CIDv1 key is already correct — never read bytes. + if gitlawb_core::cid::is_raw_cidv1(&stored) { + return Ok(()); + } + // Legacy-codec row: read the object to recompute. Counted so a test can prove + // the gate above spares non-legacy rows this read. + #[cfg(test)] + note_legacy_repair_read(); + let data = match crate::git::store::read_object(repo_path, sha)? { + Some((_ty, bytes)) => bytes, + // Bytes gone: the row stays withheld, never destructively rewritten. + None => return Ok(()), + }; + let raw = Cid::from_git_object_bytes(&data).to_string(); + if raw == stored { + return Ok(()); + } + db.repair_legacy_provider_cid(sha, &raw, &stored).await +} + +// Test-only cost-gate counter (R8, U7): how many times the opportunistic repair +// read an object's bytes on the skip path. The codec gate must spare a CIDv1/raw +// row this read; the counter is the both-ways guard (removing the gate reads the +// raw row and increments it). Same thread_local discipline as the serve-path +// oversize counter — the pin tests await `pin_new_objects` on a current-thread +// runtime, so the increment and the assertion share one thread. +#[cfg(test)] +thread_local! { + static LEGACY_REPAIR_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_legacy_repair_reads() { + LEGACY_REPAIR_READS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn legacy_repair_reads() -> usize { + LEGACY_REPAIR_READS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_legacy_repair_read() { + LEGACY_REPAIR_READS.with(|c| c.set(c.get() + 1)); +} + /// Pin a single git object to the local IPFS/Kubo node. /// /// - `ipfs_api`: base URL of the Kubo HTTP API, e.g. `http://127.0.0.1:5001`. @@ -175,6 +243,14 @@ pub async fn pin_new_objects( if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); } + // R8 (#173 round 10): opportunistically repair a legacy provider-CID + // row (Kubo dag-pb / Pinata) to the raw-content resolver key on this + // re-push. Cost-gated on the stored key's codec — a non-legacy row + // reads no bytes. Warn-only: a failure leaves the row as-is for a + // later re-push or the deferred one-shot sweep. + if let Err(e) = repair_legacy_provider_cid(repo_path, &sha, db).await { + tracing::warn!(sha = %sha, err = %e, "failed to repair legacy provider CID"); + } continue; } Ok(false) => {} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 01363abf..06b216c1 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -3271,6 +3271,335 @@ mod tests { ); } + /// Build a legacy provider CID (CIDv1 dag-pb — the Kubo above-block-size root + /// shape, and codec-equivalent to the Pinata CIDv0 legacy key for the cost + /// gate) over the object's own multihash. Non-raw codec, so `is_raw_cidv1` + /// flags it a repair candidate, and a different string from the raw key, so a + /// repair rewrites it. The existing `ipfs_cid_legacy_provider_cid_row_not_served` + /// fixture seeds a raw-codec decoy (an integrity negative the cost gate treats + /// as non-legacy on purpose); this produces the genuine dag-pb legacy shape the + /// repair path targets. Uses only the `cid` crate (already a node dep). + fn legacy_dagpb_cid(raw_cid: &str) -> String { + const DAG_PB: u64 = 0x70; + let parsed = raw_cid + .parse::>() + .expect("the raw CID parses"); + cid::CidGeneric::<64>::new_v1(DAG_PB, *parsed.hash()).to_string() + } + + /// #173 R8 (jatmn round 10, U7 — load-bearing): a legacy row keyed on a PROVIDER + /// CID (Kubo dag-pb / Pinata) is opportunistically rewritten to the raw-content + /// key on a re-push whose pack carries the object, stashing the old value in + /// `legacy_provider_cid`. The advertised key 404s while the row is legacy (the + /// resolver recomputes the raw CID and the stored key does not match) and serves + /// after repair. RED before the skip-branch repair lands (the raw key 404s post + /// pin). Also asserts the repair leaves `pinata_cid` NULL (scenario 3) and that + /// the retired provider CID still refuses to serve (scenario 6, integrity). + #[sqlx::test] + async fn ipfs_cid_legacy_provider_cid_repaired_on_repush(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["provsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provsrc.git"); + let repo = seed_repo(&owner_did, "provsrc"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + + // The canonical raw key the resolver accepts once the row is repaired. + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes( + &crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .unwrap() + .1, + ) + .to_string(); + // The key stored today: a genuine legacy dag-pb provider CID. + let provider_cid = legacy_dagpb_cid(&raw_cid); + assert_ne!( + provider_cid, raw_cid, + "the provider CID differs from the raw resolver key" + ); + + // Legacy-shape row: cid = the PROVIDER CID (raw SQL — the helpers store the + // raw CID). The object itself is public and servable. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&fx.public_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + // RED baseline: the raw key a correct client sends 404s while the row is legacy. + let (st_before, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st_before, + StatusCode::OK, + "the raw key 404s while the row is keyed on the provider CID" + ); + + // Re-push carries the object again: `pin_new_objects` hits the already-pinned + // skip branch and repairs the row. The `/add` mock must NOT fire — the object + // is already on IPFS, never re-pinned. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + vec![fx.public_oid.clone()], + &state.db, + &repo.id, + ) + .await; + m.assert_async().await; + + // GREEN: the key is repaired to the raw CID and the old value is stashed. + let (stored_cid, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(&fx.public_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + stored_cid, raw_cid, + "the key is repaired to the raw-content CID" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed in legacy_provider_cid" + ); + + // The advertised (raw) key now serves 200. + let (st_after, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st_after, + StatusCode::OK, + "the repaired raw key serves after the re-push" + ); + assert!(body.contains("public bytes"), "the object's bytes serve"); + + // Scenario 3: repair never wrote `pinata_cid`, so the Pinata pin-skip gate + // (`has_pinata_cid`) is untouched and Pinata still pins the object. + assert!( + !state.db.has_pinata_cid(&fx.public_oid).await.unwrap(), + "repair leaves pinata_cid NULL" + ); + + // Scenario 6 (integrity negative): the retired provider CID still 404s — no + // serve-path alias for a CID the bytes do not hash to. + let (st_old, body_old) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&provider_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st_old, + StatusCode::OK, + "the retired provider CID must not serve after repair" + ); + assert!( + !body_old.contains("public bytes"), + "no bytes egress under the retired provider CID" + ); + } + + /// #173 R8 (U7 cost gate): a well-formed CIDv1/raw already-pinned row triggers NO + /// object read on the skip path — the codec check decides candidacy from the + /// stored string alone, so a non-legacy row keeps the DB-only skip cost. Also + /// covers the small-object equivalence: a small legacy object Kubo pins under the + /// raw key (raw-leaves) is already CIDv1/raw and needs no repair. The read counter + /// is the both-ways guard: removing the codec gate reads the raw row and trips it. + #[sqlx::test] + async fn ipfs_cid_repair_codec_gate_skips_raw_row(pool: PgPool) { + let state = test_state(pool).await; + let fx = seed_cid_repos("codecgate", "cg", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("codecgate") + .join("pinsrc.git"); + + // A correct raw-CID row (steady state), recorded via the production helper. + let raw_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + assert!( + gitlawb_core::cid::is_raw_cidv1(&raw_cid), + "the helper records a CIDv1/raw key" + ); + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"x"}"#) + .expect(0) + .create_async() + .await; + + crate::ipfs_pin::reset_legacy_repair_reads(); + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + vec![fx.public_oid.clone()], + &state.db, + "repoCG", + ) + .await; + m.assert_async().await; + + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a CIDv1/raw row triggers no object read on the skip path (cost gate)" + ); + assert_eq!( + state + .db + .cid_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some(raw_cid.as_str()), + "the raw row is left as-is" + ); + } + + /// #173 R8 (U7): a legacy row whose object bytes are gone stays withheld — the + /// repair never destructively rewrites it, so the row is preserved for a future + /// re-push or the deferred one-shot sweep. + #[sqlx::test] + async fn ipfs_cid_repair_unrepairable_row_stays_withheld(pool: PgPool) { + let state = test_state(pool.clone()).await; + let _fx = seed_cid_repos("unrep", "ur", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("unrep") + .join("pinsrc.git"); + + // A legacy dag-pb row for an oid whose bytes are NOT in this bare repo. + let phantom_oid = "b".repeat(64); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind(&phantom_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + let mut server = mockito::Server::new_async().await; + server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"x"}"#) + .expect(0) + .create_async() + .await; + + // Skip-branch runs (is_pinned true) but read_object returns None (bytes gone), + // so the repair returns without touching the row. + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + vec![phantom_oid.clone()], + &state.db, + "repoUR", + ) + .await; + + let (stored, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(&phantom_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + stored, provider_cid, + "an unrepairable row keeps its provider CID (no destructive rewrite)" + ); + assert_eq!( + stashed, None, + "no legacy_provider_cid is stashed when the bytes are gone" + ); + } + + /// #173 R8 (U7, INV-7 upgrade path): a node already at the prior-max schema (v13) + /// gets `pinned_cids.legacy_provider_cid` from the NEW v14 migration. Simulate the + /// pre-v14 node by dropping the column and un-applying v14, then re-migrate and + /// assert a repair round-trips through the column. RED before the v14 migration + /// exists (the column is never re-added → the repair UPDATE errors). + #[sqlx::test] + async fn pinned_cids_legacy_provider_cid_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v14 shape: drop the column and forget v14 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS legacy_provider_cid") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 14") + .execute(&pool) + .await + .unwrap(); + + // Upgrade: re-run migrations → v14 re-adds the column. + state.db.run_migrations().await.expect("migrate to v14"); + + // A repair round-trips through the v14 column. + state + .db + .record_pinned_cid("upg_oid", "QmProviderLegacy", None) + .await + .unwrap(); + state + .db + .repair_legacy_provider_cid("upg_oid", "bRawContentKey", "QmProviderLegacy") + .await + .unwrap(); + let (cid, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = 'upg_oid'", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(cid, "bRawContentKey", "v14 lets the repair rewrite the key"); + assert_eq!( + stashed.as_deref(), + Some("QmProviderLegacy"), + "the v14 legacy_provider_cid column is present after upgrade" + ); + } + /// #173 (provenance-path throttle): a walk-requiring provenanced candidate whose /// per-IP walk quota is spent returns 429 (the provenance arm's Throttled outcome, /// then the fall-through). quota=1, keyed on XFF. The first reader request runs the From 9cef54ea1815beec436d58181ff96bf955f3860a Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 20 Jul 2026 20:27:35 -0500 Subject: [PATCH 14/77] fix(review): share one deadline across the /ipfs size+content reads Code review found each candidate's size and content reads were each granted a full git_service_timeout, so a single served candidate could hold the /ipfs walk permit for 2x the timeout. Share one deadline across both stages (mirrors build_filtered_pack) so the read holds admission for one timeout total, and correct the admission-bound comment: the worst case is O(candidates x timeout) bounded by the walk concurrency cap, not a single deadline. --- crates/gitlawb-node/src/api/ipfs.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index c5fa35bc..624a1b27 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -196,9 +196,13 @@ pub async fn get_by_cid( // instant the handler future is torn down while a spawn_blocking git probe/walk/read // is still alive (the disconnect-spam cap bypass this closes, the /ipfs half of #174 // P1-a). Every git child on this pipeline is duration-bounded (the run_bounded_git - // probe/read twins + the bounded walk), so the detached task cannot hold admission - // past ~git_service_timeout_secs. The client key is already captured (`source_key`) - // before the spawn; the detached task has no request extractors. + // probe/read twins + the bounded walk), and each candidate's size+content read shares + // one deadline; so admission is bounded per candidate. The legacy scan can still + // iterate up to `ipfs_max_legacy_probes` candidates serially, so the worst-case hold + // is O(candidates x git_service_timeout_secs), NOT a single timeout — what actually + // bounds cancel-spam is the walk concurrency cap (global + per-source), not the + // per-child deadline. The client key is already captured (`source_key`) before the + // spawn; the detached task has no request extractors. let admission = crate::git::smart_http::AdmissionGuard::new(ipfs_walk_permit, ipfs_caller_permit); let serve: tokio::task::JoinHandle> = tokio::spawn(async move { @@ -759,9 +763,14 @@ async fn gate_and_serve( // deadline rather than left to pin the held /ipfs walk permit (#173 round-10, KTD2). // No outer timeout, mirroring the bounded walk; a `GitServiceTimeout` from either // twin surfaces as `ServedRead::ReadErr` -> truncated (retryable 503). - let read_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + // ONE deadline spans the size and content reads, so a single served candidate + // holds the /ipfs walk permit for at most `git_service_timeout_secs` total, not + // one full timeout per stage (mirrors `build_filtered_pack`'s shared deadline). + let read_deadline = std::time::Instant::now() + + std::time::Duration::from_secs(state.config.git_service_timeout_secs); let read = tokio::task::spawn_blocking(move || -> ServedRead { - match store::object_size_bounded(&git_bin, &read_repo, &read_sha, read_timeout) { + let size_budget = read_deadline.saturating_duration_since(std::time::Instant::now()); + match store::object_size_bounded(&git_bin, &read_repo, &read_sha, size_budget) { Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), Ok(Some(_)) => {} // git ran and reported no such object (or an unparseable size): genuine @@ -771,12 +780,13 @@ async fn gate_and_serve( // infra/timeout failure, not a not-found. Err(e) => return ServedRead::ReadErr(e.to_string()), } + let content_budget = read_deadline.saturating_duration_since(std::time::Instant::now()); let content = match store::read_object_content_bounded( &git_bin, &read_repo, &read_sha, &read_type, - read_timeout, + content_budget, ) { Ok(c) => c, Err(e) => return ServedRead::ReadErr(e.to_string()), From cc41d37befa340cc99f2fa51b3e83f863dfab34d Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 20 Jul 2026 21:06:10 -0500 Subject: [PATCH 15/77] fix(node): bound the post-push pin and repair reads so a wedge cannot pin the coalescing key A cross-model adversarial pass found that run_post_push_replication holds the per-repo coalescing key until requeue_or_release, but pin_new_objects reached it only after two unbounded git reads: the U7 legacy-repair read and the pre-existing pin read, both plain Command::output with no teardown. A wedged cat-file on a stuck backend hung the task forever, so the key was held until process death and later pushes only marked it dirty without spawning a replacement (the same class this PR closed on the /ipfs serve path). Add read_object_bounded (the bounded twins under one shared deadline) and use it at both sites; a timeout skips that object and lets the task proceed to requeue_or_release. Adds a wedge-reap regression proven load-bearing two ways. --- crates/gitlawb-node/src/api/repos.rs | 2 + crates/gitlawb-node/src/git/store.rs | 36 +++++++ crates/gitlawb-node/src/ipfs_pin.rs | 45 +++++--- crates/gitlawb-node/src/test_support.rs | 131 ++++++++++++++++++++++++ 4 files changed, 201 insertions(+), 13 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index c9a2b3dd..08075e4e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -885,6 +885,8 @@ async fn run_post_push_replication( crate::ipfs_pin::pin_new_objects( &ctx.ipfs_api, &ctx.disk_path, + &ctx.git_bin, + ctx.timeout, object_list.clone(), &ctx.db, &ctx.repo_id, diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index a79d596a..899394ee 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -407,6 +407,42 @@ pub fn read_object(repo_path: &Path, sha256_hex: &str) -> Result Result)>> { + let deadline = std::time::Instant::now() + timeout; + let obj_type = match object_type_bounded( + git_bin, + repo_path, + sha256_hex, + deadline.saturating_duration_since(std::time::Instant::now()), + )? { + Some(t) => t, + None => return Ok(None), + }; + let content = read_object_content_bounded( + git_bin, + repo_path, + sha256_hex, + &obj_type, + deadline.saturating_duration_since(std::time::Instant::now()), + )?; + Ok(Some((obj_type, content))) +} + /// Get the diff between two branches: changes on source_branch not in target_branch. pub fn branch_diff(repo_path: &Path, target_branch: &str, source_branch: &str) -> Result { let output = Command::new("git") diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 3cf5610c..3d2f13df 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -63,6 +63,8 @@ where /// recompute. A row whose bytes are gone stays withheld (no destructive rewrite). async fn repair_legacy_provider_cid( repo_path: &std::path::Path, + git_bin: &str, + git_timeout: Duration, sha: &str, db: &crate::db::Db, ) -> Result<()> { @@ -78,10 +80,18 @@ async fn repair_legacy_provider_cid( // the gate above spares non-legacy rows this read. #[cfg(test)] note_legacy_repair_read(); - let data = match crate::git::store::read_object(repo_path, sha)? { - Some((_ty, bytes)) => bytes, + let data = match crate::git::store::read_object_bounded(git_bin, repo_path, sha, git_timeout) { + Ok(Some((_ty, bytes))) => bytes, // Bytes gone: the row stays withheld, never destructively rewritten. - None => return Ok(()), + Ok(None) => return Ok(()), + // A wedged/D-state `git cat-file` (timeout/infra): the repair is opportunistic + // and best-effort, so skip it and return Ok so the pin task PROCEEDS to + // requeue_or_release rather than hanging the coalescing key until process death + // (grok F2, #173). A later re-push or the deferred sweep retries the repair. + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "skipping legacy provider-CID repair: bounded object read failed"); + return Ok(()); + } }; let raw = Cid::from_git_object_bytes(&data).to_string(); if raw == stored { @@ -204,6 +214,8 @@ pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { pub async fn pin_new_objects( ipfs_api: &str, repo_path: &std::path::Path, + git_bin: &str, + git_timeout: Duration, object_list: Vec, db: &crate::db::Db, repo_id: &str, @@ -248,7 +260,9 @@ pub async fn pin_new_objects( // re-push. Cost-gated on the stored key's codec — a non-legacy row // reads no bytes. Warn-only: a failure leaves the row as-is for a // later re-push or the deferred one-shot sweep. - if let Err(e) = repair_legacy_provider_cid(repo_path, &sha, db).await { + if let Err(e) = + repair_legacy_provider_cid(repo_path, git_bin, git_timeout, &sha, db).await + { tracing::warn!(sha = %sha, err = %e, "failed to repair legacy provider CID"); } continue; @@ -260,15 +274,20 @@ pub async fn pin_new_objects( } } - // Read raw object content - let data = match crate::git::store::read_object(repo_path, &sha) { - Ok(Some((_obj_type, bytes))) => bytes, - Ok(None) => continue, - Err(e) => { - tracing::warn!(sha = %sha, err = %e, "failed to read git object for pinning"); - continue; - } - }; + // Read raw object content under a bounded read so a wedged/D-state `git + // cat-file` (stuck NFS/Tigris backend) is reaped at `git_timeout` instead of + // hanging pin_new_objects forever — which would pin the post-push coalescing + // key until process death (grok F2, #173). On Err the object is simply not + // pinned this pass; a later pass/push retries. + let data = + match crate::git::store::read_object_bounded(git_bin, repo_path, &sha, git_timeout) { + Ok(Some((_obj_type, bytes))) => bytes, + Ok(None) => continue, + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "failed to read git object for pinning"); + continue; + } + }; // Pin to IPFS match pin_git_object(ipfs_api, &sha, &data).await { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 06b216c1..0d0ae2e8 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -2430,6 +2430,8 @@ mod tests { crate::ipfs_pin::pin_new_objects( &server.url(), &pub_bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, &pub_repo.id, @@ -2553,6 +2555,8 @@ mod tests { crate::ipfs_pin::pin_new_objects( &server.url(), &pub_bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, &pub_repo.id, @@ -3120,6 +3124,8 @@ mod tests { let pinned = crate::ipfs_pin::pin_new_objects( &server.url(), &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, "repoZ", @@ -3142,6 +3148,123 @@ mod tests { ); } + /// #173 (grok F2): the post-push pin read is BOUNDED, so a wedged/D-state + /// `git cat-file` (stuck NFS/Tigris backend) is reaped at `git_timeout` and + /// `pin_new_objects` RETURNS — reaching `requeue_or_release` in production — + /// instead of hanging forever and pinning the per-repo coalescing key until + /// process death. A fake `git` whose `cat-file` records its pid then sleeps far + /// past a SHORT 1s timeout stands in for the wedged backend; the `run_bounded_git` + /// watchdog (SIGTERM -> grace -> SIGKILL of the process group) must reap it well + /// before its 8s natural exit, and the call must return with nothing pinned. + /// + /// REVERT PROOF (RED): swap `read_object_bounded` back to the bare + /// `store::read_object` at the pin read and the wedged child is STILL RUNNING at + /// the mid-flight liveness poll below (unbounded `Command::output` cannot be + /// reaped at the deadline) — the reap assertion fails. + #[cfg(unix)] + #[sqlx::test] + async fn pin_new_objects_reaps_wedged_read_at_deadline(pool: PgPool) { + use std::time::Duration; + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + // Fake `git`: `cat-file` records its own pid then sleeps 8s (>> the 1s + // deadline) so the read is genuinely wedged; the watchdog is what must end it. + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("catfile.pid"); + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + cat-file) echo $$ > \"{}\"; sleep 8 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pidfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let repo = tmp.path().to_path_buf(); + let git = git_path.to_str().unwrap().to_string(); + // A never-pinned OID so the call reaches the object-read stage (not the + // already-pinned skip path). + let oid = "f".repeat(64); + // Non-empty ipfs_api so `pin_new_objects` does not early-return; the wedged + // read is reaped and the OID skipped before any `/add`, so this URL is unused. + let ipfs_api = "http://127.0.0.1:1".to_string(); + + // `pin_new_objects` must run on THIS runtime so its `is_pinned` DB call keeps + // the sqlx pool on its home runtime. The bounded read is a synchronous blocking + // call, so the reap poll runs on a separate OS thread (independent of tokio): it + // captures the wedged child's pid, waits past the deadline, records whether it + // was reaped, then SIGKILLs defensively so even a true infinite hang cannot leak + // an orphan or stall the awaited call. + let pidfile_poll = pidfile.clone(); + let poll = std::thread::spawn(move || -> (Option, bool) { + let alive = |pid: i32| unsafe { libc::kill(pid, 0) == 0 }; + let mut pid = None; + for _ in 0..500 { + if let Some(p) = std::fs::read_to_string(&pidfile_poll) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + pid = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let pid = match pid { + Some(p) => p, + None => return (None, false), + }; + // Past the 1s deadline + SIGTERM grace but well before the 8s natural exit: + // the bounded read must already have reaped the wedged group. The unbounded + // `store::read_object` leaves it running here — the load-bearing RED. + std::thread::sleep(Duration::from_secs(3)); + let reaped = !alive(pid); + unsafe { + libc::kill(pid, libc::SIGKILL); + } + (Some(pid), reaped) + }); + + // The call must RETURN — reaching `requeue_or_release` in production — rather + // than hang on the 8s sleep. The poll thread's defensive SIGKILL guarantees the + // read completes even in the unbounded RED case, so this observes a bounded + // return either way; the reap assertion below is what separates RED from GREEN. + let pinned = tokio::time::timeout( + Duration::from_secs(6), + crate::ipfs_pin::pin_new_objects( + &ipfs_api, + &repo, + &git, + Duration::from_secs(1), + vec![oid], + &db, + "repoWedge", + ), + ) + .await + .expect("pin_new_objects must return within the bound, not hang on the wedged read"); + + let (pid, reaped) = poll.join().expect("poll thread joins"); + pid.expect("the fake cat-file must have spawned and recorded its pid"); + assert!( + reaped, + "the post-push pin read must reap the wedged cat-file child at the deadline, \ + not leave it running (which would pin the coalescing key until process death)" + ); + assert!( + pinned.is_empty(), + "a wedged read pins nothing this pass; a later pass/push retries" + ); + } + /// #173 (jatmn, F2): a legacy pin with NULL provenance backfills its source /// via `backfill_pin_provenance`, and the `AND repo_id IS NULL` guard preserves /// first-pinner-owns (a non-NULL provenance is left untouched). @@ -3248,6 +3371,8 @@ mod tests { let pinned = crate::ipfs_pin::pin_new_objects( &server.url(), &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, "repoBF", @@ -3367,6 +3492,8 @@ mod tests { crate::ipfs_pin::pin_new_objects( &server.url(), &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, &repo.id, @@ -3468,6 +3595,8 @@ mod tests { crate::ipfs_pin::pin_new_objects( &server.url(), &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, "repoCG", @@ -3530,6 +3659,8 @@ mod tests { crate::ipfs_pin::pin_new_objects( &server.url(), &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![phantom_oid.clone()], &state.db, "repoUR", From a4f7d5dffbbc81b201a7959e1e2526d84da64173 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:22:45 -0500 Subject: [PATCH 16/77] fix(node): own the repo write lock on one connection so cancellation cannot leak it `acquire_write` took `pg_try_advisory_lock` through the shared pool, then awaited the Tigris exists/download path before constructing `RepoWriteGuard`. Two bugs fell out of that. The `tokio::time::timeout` this branch added around the call can drop the future inside that window, after the lock is held and before the only unlock (`RepoWriteGuard::release`) exists, wedging every later push to the repo. And because a session advisory lock belongs to the connection that took it, running the lock and the unlock through the pool lets them land on different connections: the unlock returns false and leaks, while a competing acquire that draws the holding connection re-enters the lock and two writers run against the same repo concurrently. `RepoStore` now owns a dedicated lock pool built with an `after_release` hook that runs `pg_advisory_unlock_all()`. `acquire_write` checks out one connection, takes the lock on it, and moves it into the guard, which unlocks on that same connection. sqlx's `PoolConnection::drop` spawns `return_to_pool`, which runs the hook, so a connection dropped by cancellation still clears its locks; the unlock is asynchronous with respect to the drop, which the tests account for. The pool is separate from the main query pool because a push holds its connection for the whole receive-pack and `db_max_connections` (20) sits below `max_concurrent_git_pushes` (32). Seven tests cover the cancellation window, the same-session release, and the serialization the pool-mismatch bug was breaking. Both the cancellation regression and the concurrent-writer case were observed red before the fix, and removing the `after_release` hook or moving the unlock back to the pool turns them red again. --- crates/gitlawb-node/src/api/repos.rs | 5 +- crates/gitlawb-node/src/git/repo_store.rs | 452 +++++++++++++++++++++- crates/gitlawb-node/src/main.rs | 16 +- 3 files changed, 454 insertions(+), 19 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 08075e4e..b817318d 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1490,7 +1490,10 @@ pub async fn git_receive_pack( // exhausted Postgres pool (so the 60-count never advances) — and the write permit // is held the whole time, draining the pool (#174 P1-2). The outer // `tokio::time::timeout` cancels a mid-sleep/mid-`fetch_one` future, so it bounds - // both the loop and a hung iteration without any repo_store.rs change (KTD3). The + // both the loop and a hung iteration. Cancelling here is only safe because + // acquire_write holds its advisory lock on a connection from a pool whose + // `after_release` hook unlocks (#173): the dropped future used to leave the lock + // held with no guard alive to release it, wedging later pushes to that repo. The // permit is a handler local here (moved into the AdmissionGuard only after this), // so the early return on timeout drops it and frees the slot; shed a bounded 503. let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index a5c367e9..16c0378b 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -11,9 +11,12 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::time::Duration; use anyhow::{Context, Result}; -use sqlx::PgPool; +use sqlx::pool::PoolConnection; +use sqlx::postgres::PgPoolOptions; +use sqlx::{PgPool, Postgres}; use tokio::sync::Mutex; use tracing::{debug, info, warn}; @@ -25,30 +28,54 @@ use super::tigris::TigrisClient; pub struct RepoStore { repos_dir: PathBuf, tigris: Option, - /// Shared Postgres pool for advisory locks. - pool: PgPool, + /// Dedicated Postgres pool for repo write advisory locks, built by + /// `build_lock_pool` (see there for why it is separate and why it carries an + /// `after_release` hook). Never use this for ordinary queries. + lock_pool: PgPool, /// Tracks repos already confirmed to exist in Tigris — avoids redundant /// HEAD checks and background uploads for repos we've already migrated. migrated: Arc>>, + /// Test-only stall injected at the head of `acquire_write`'s Tigris phase, + /// i.e. AFTER the advisory lock is taken and BEFORE the guard exists. That + /// window is exactly where the outer `tokio::time::timeout` in + /// `api/repos.rs` can drop the future (#173). `TigrisClient` takes its + /// endpoint from process-wide AWS env vars and has no injectable seam, so + /// this flag is the smallest way to hold a real `acquire_write` open in that + /// window and cancel it there. + #[cfg(test)] + tigris_stall: Option, } impl RepoStore { + /// Derives its own lock pool from `pool`, so callers that only have the main + /// pool (tests, `for_testing` sites in other modules) still get the + /// `after_release` semantics `acquire_write` depends on. #[cfg(test)] pub fn for_testing(repos_dir: PathBuf, pool: PgPool) -> Self { - Self { + Self::new( repos_dir, - tigris: None, - pool, - migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), - } + None, + build_lock_pool(&pool, 8, Duration::from_secs(5)), + ) + } + + /// Test-only: see `tigris_stall`. + #[cfg(test)] + pub fn with_tigris_stall(mut self, stall: Duration) -> Self { + self.tigris_stall = Some(stall); + self } - pub fn new(repos_dir: PathBuf, tigris: Option, pool: PgPool) -> Self { + /// `lock_pool` must come from `build_lock_pool`; a plain pool leaks advisory + /// locks on cancellation. + pub fn new(repos_dir: PathBuf, tigris: Option, lock_pool: PgPool) -> Self { Self { repos_dir, tigris, - pool, + lock_pool, migrated: Arc::new(Mutex::new(HashSet::new())), + #[cfg(test)] + tigris_stall: None, } } @@ -157,13 +184,34 @@ impl RepoStore { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); + // Check out ONE connection from the lock pool and keep it for the whole + // lock lifetime. Two reasons, both bugs we hit with `fetch_one(&pool)`: + // + // * A session-level advisory lock belongs to the CONNECTION that took + // it. Running the lock and the unlock through the pool lets them land + // on different connections, so `pg_advisory_unlock` silently returns + // false and the lock leaks, while a competing acquire that happens to + // draw the holding connection re-enters the lock and pushes to the + // same repo run concurrently. + // * Cancellation. `api/repos.rs` bounds this call with + // `tokio::time::timeout`; when it fires during the Tigris phase below + // the future is dropped after the lock was taken and before + // `RepoWriteGuard` (the only caller of `pg_advisory_unlock`) exists. + // Dropping this connection instead runs the pool's `after_release` + // hook, which clears the lock (#173). + let mut lock_conn = self + .lock_pool + .acquire() + .await + .context("checking out a lock-pool connection")?; + // Acquire Postgres advisory lock with retry using pg_try_advisory_lock // to avoid blocking indefinitely on stale locks from crashed connections. let mut acquired = false; for attempt in 0..60 { let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(lock_key) - .fetch_one(&self.pool) + .fetch_one(&mut *lock_conn) .await .context("trying advisory lock")?; if row.0 { @@ -178,6 +226,11 @@ impl RepoStore { anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}"); } + #[cfg(test)] + if let Some(stall) = self.tigris_stall { + tokio::time::sleep(stall).await; + } + // Always download the latest from Tigris before writing. // Local disk may be stale if another machine pushed since our last access. if let Some(ref tigris) = self.tigris { @@ -202,7 +255,7 @@ impl RepoStore { repo_name: repo_name.to_string(), local_path, lock_key, - pool: self.pool.clone(), + lock_conn, tigris: self.tigris.clone(), }) } @@ -354,7 +407,12 @@ pub struct RepoWriteGuard { repo_name: String, pub local_path: PathBuf, lock_key: i64, - pool: PgPool, + /// The lock-pool connection that TOOK the advisory lock. It must be the one + /// that releases it (session locks are owned by their connection), and + /// holding it here is also what makes a guard dropped without `release` + /// safe: the drop returns the connection through `after_release`, which + /// clears the lock. + lock_conn: PoolConnection, tigris: Option, } @@ -369,7 +427,7 @@ impl RepoWriteGuard { /// half-applied or otherwise inconsistent repo would propagate corruption to /// Tigris (and to every node that later downloads it). The lock is always /// released regardless, to avoid stale locks blocking future writes. - pub async fn release(self, success: bool) { + pub async fn release(mut self, success: bool) { // Upload to Tigris only on success. if success { if let Some(ref tigris) = self.tigris { @@ -384,14 +442,56 @@ impl RepoWriteGuard { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } - // Release advisory lock + // Release the advisory lock on the connection that took it. Anything else + // (a fresh `&pool` checkout) is a no-op that returns false: Postgres + // scopes a session lock to its owning connection. let _ = sqlx::query("SELECT pg_advisory_unlock($1)") .bind(self.lock_key) - .execute(&self.pool) + .execute(&mut *self.lock_conn) .await; + // Dropping `self` returns the connection to the lock pool, where + // `after_release` sweeps anything the unlock above missed. } } +/// Build the dedicated advisory-lock pool a `RepoStore` runs its write locks on. +/// Connect options are cloned off an existing pool so callers need not re-parse +/// the database URL; the pool is lazy, so no connection is opened here. +/// +/// Two properties, both load-bearing: +/// +/// * The `after_release` hook runs `pg_advisory_unlock_all()` before a +/// connection goes back into the pool. sqlx's `PoolConnection::drop` spawns +/// `return_to_pool()`, which invokes this hook, so a connection dropped by +/// CANCELLATION still clears its locks. That is what keeps an `acquire_write` +/// killed mid-Tigris by the caller's `tokio::time::timeout` from leaking a +/// lock and wedging every later push to that repo (#173). Note the hook runs +/// from that spawned task, so the unlock is asynchronous with respect to the +/// drop: the lock clears shortly after the connection goes away, not +/// synchronously with it. +/// * It is a SEPARATE pool from the main query pool, not a slice of it. A push +/// holds its lock connection for the whole receive-pack, and +/// `db_max_connections` (default 20) is well below +/// `max_concurrent_git_pushes` (default 32), so drawing these from the main +/// pool would starve every other query during a push burst. +/// +/// `acquire_timeout` bounds the wait when every lock-pool connection is busy, so +/// exhaustion surfaces as a clean error rather than an unbounded hang. +pub fn build_lock_pool(source: &PgPool, max_connections: u32, acquire_timeout: Duration) -> PgPool { + PgPoolOptions::new() + .max_connections(max_connections) + .acquire_timeout(acquire_timeout) + .after_release(|conn, _meta| { + Box::pin(async move { + sqlx::query("SELECT pg_advisory_unlock_all()") + .execute(&mut *conn) + .await?; + Ok(true) + }) + }) + .connect_lazy_with((*source.connect_options()).clone()) +} + /// Compute a stable i64 hash for a Postgres advisory lock key. fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { use std::hash::{Hash, Hasher}; @@ -404,6 +504,326 @@ fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { #[cfg(test)] mod tests { use super::*; + use std::time::Duration; + + // ── advisory-lock test helpers (#173 U1) ─────────────────────────────── + + /// Postgres advisory locks live in a CLUSTER-wide space, not a per-database + /// one, so two `#[sqlx::test]` cases running against their own temporary + /// databases still share the key space. Every lock test therefore mints its + /// own key instead of reusing a fixed constant. + fn unique_lock_key() -> i64 { + use std::sync::atomic::{AtomicI64, Ordering}; + static NEXT: AtomicI64 = AtomicI64::new(0); + let n = NEXT.fetch_add(1, Ordering::Relaxed); + ((std::process::id() as i64) << 24) | (n & 0xff_ffff) + } + + /// A plain pool (no `after_release` hook, no idle timeout) at the same + /// database as the `#[sqlx::test]` pool. Two separate reasons these tests + /// cannot just use the pool the harness hands them: + /// + /// 1. Observing lock state has to happen from a session that is definitely + /// not the one under test. Session advisory locks are re-entrant, so + /// `pg_try_advisory_lock` on the very connection that already holds the key + /// returns true, and a same-pool probe silently reports a leaked lock free. + /// 2. The harness pool sets `idle_timeout(1s)`, so a connection returned to it + /// is closed about a second later and Postgres drops every lock that + /// session held. That would mask exactly the leak these tests exist to + /// catch, so the store under test runs on one of these too. + fn sibling_pool(pool: &PgPool, max_connections: u32) -> PgPool { + sqlx::postgres::PgPoolOptions::new() + .max_connections(max_connections) + .connect_lazy_with((*pool.connect_options()).clone()) + } + + /// Probe the lock from a connection that is NOT the one under test. Session + /// advisory locks are re-entrant within their own session, so a check from the + /// holding connection would pass vacuously and prove nothing. + async fn lock_is_free_elsewhere(pool: &PgPool, key: i64) -> bool { + let mut probe = pool.acquire().await.expect("probe connection"); + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *probe) + .await + .expect("probe try-lock"); + if taken.0 { + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *probe) + .await + .expect("probe unlock"); + } + taken.0 + } + + /// `after_release` runs from the task sqlx spawns in `PoolConnection::drop`, + /// so the unlock is ASYNCHRONOUS with respect to the drop. Callers must poll + /// rather than assume the lock is gone the instant the connection goes away. + async fn wait_until_free(pool: &PgPool, key: i64, within: Duration) -> bool { + let deadline = std::time::Instant::now() + within; + loop { + if lock_is_free_elsewhere(pool, key).await { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + } + + // ── DESIGN GATE ──────────────────────────────────────────────────────── + // The whole cancellation-safety design rests on one sqlx behaviour: + // `PoolConnection::drop` spawns `return_to_pool()`, which invokes the pool's + // `after_release` hook before the connection is reused. If that holds, a + // connection dropped by cancellation still runs `pg_advisory_unlock_all()` + // and the lock cannot leak. This test proves it by execution, through the + // production `build_lock_pool` so that stripping the hook there turns it red. + + #[sqlx::test] + async fn dropped_pool_connection_runs_after_release_and_clears_locks(pool: PgPool) { + let key = unique_lock_key(); + let lock_pool = build_lock_pool(&pool, 4, Duration::from_secs(5)); + + { + let mut conn = lock_pool.acquire().await.expect("lock-pool connection"); + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *conn) + .await + .expect("try-lock"); + assert!(taken.0, "first try-lock must succeed"); + assert!( + !lock_is_free_elsewhere(&pool, key).await, + "lock must be observably HELD from another session while the connection lives" + ); + // Drop WITHOUT calling pg_advisory_unlock: this models cancellation. + } + + assert!( + wait_until_free(&pool, key, Duration::from_secs(5)).await, + "after_release must clear the advisory lock of a dropped connection" + ); + } + + // ── acquire_write cancellation safety (#173 U1) ──────────────────────── + + /// The reviewer's named regression. `api/repos.rs` wraps `acquire_write` in a + /// `tokio::time::timeout`; when that fires during the Tigris phase the future + /// is dropped after the advisory lock was taken and before `RepoWriteGuard` + /// (the only thing that unlocks) exists. The lock then leaks and every later + /// push to the same repo spins the 60-attempt / 60s ceiling and fails. + #[sqlx::test] + async fn cancelled_acquire_write_mid_tigris_does_not_leak_the_lock(pool: PgPool) { + let repos_dir = PathBuf::from("/tmp/gitlawb-test-repos"); + let owner = "did:key:z6MkCancelMidTigris"; + let repo = "cancel-mid-tigris"; + + let store_pool = sibling_pool(&pool, 8); + let stalling = RepoStore::for_testing(repos_dir.clone(), store_pool.clone()) + .with_tigris_stall(Duration::from_secs(30)); + let cancelled = tokio::time::timeout( + Duration::from_millis(500), + stalling.acquire_write(owner, repo), + ) + .await; + assert!( + cancelled.is_err(), + "the acquire must still be inside the Tigris phase when the timeout fires" + ); + + // Observed from an independent session, so the check cannot be satisfied + // by re-entrancy on whichever pooled connection happens to be handed back. + let probe = sibling_pool(&pool, 2); + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + assert!( + wait_until_free(&probe, key, Duration::from_secs(5)).await, + "a cancelled acquire_write must leave no advisory lock held" + ); + + // A subsequent acquire for the SAME repo must succeed promptly. Before the + // fix it blocks on the leaked lock until the 60-attempt ceiling. + let store = RepoStore::for_testing(repos_dir, store_pool); + let guard = tokio::time::timeout(Duration::from_secs(5), store.acquire_write(owner, repo)) + .await + .expect("second acquire_write must not block on a leaked lock") + .expect("second acquire_write must succeed"); + guard.release(false).await; + } + + /// Cancellation BEFORE the lock is taken must leave nothing behind: no lock, + /// and no lock-pool connection stranded. The lock pool here holds exactly one + /// connection, so a stranded one would make the follow-up acquire time out + /// waiting for a checkout. + #[sqlx::test] + async fn cancelled_acquire_write_before_the_lock_leaves_nothing_held(pool: PgPool) { + let probe = sibling_pool(&pool, 2); + let owner = "did:key:z6MkCancelEarly"; + let repo = "cancel-early"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 1, Duration::from_secs(3)), + ); + + // A zero deadline polls the future once, which gets it no further than the + // first await (the pool checkout / the first try-lock round trip), so it is + // cancelled before any lock can be taken. + let cancelled = + tokio::time::timeout(Duration::ZERO, store.acquire_write(owner, repo)).await; + assert!(cancelled.is_err(), "the acquire must be cancelled"); + + assert!( + lock_is_free_elsewhere(&probe, key).await, + "no lock may be held when the acquire never got that far" + ); + + // The single lock-pool connection must be back: if cancellation stranded + // it, this checkout blocks until the 3s acquire timeout and fails. + let guard = tokio::time::timeout(Duration::from_secs(2), store.acquire_write(owner, repo)) + .await + .expect("the lock-pool connection must have been returned") + .expect("acquire after cancellation"); + guard.release(false).await; + } + + /// Lock-pool exhaustion is a bounded wait and a clean error, never a panic and + /// never an unbounded hang. + #[sqlx::test] + async fn lock_pool_exhaustion_is_a_bounded_error(pool: PgPool) { + let owner = "did:key:z6MkExhaustion"; + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 1, Duration::from_secs(2)), + ); + + let held = store + .acquire_write(owner, "exhaust-a") + .await + .expect("first acquire"); + + // Different repo, so this is not the advisory lock queueing: the only + // connection in the lock pool is checked out by `held`. + let started = std::time::Instant::now(); + let err = tokio::time::timeout( + Duration::from_secs(10), + store.acquire_write(owner, "exhaust-b"), + ) + .await + .expect("the wait must be bounded by the pool acquire timeout"); + let err = match err { + Ok(_) => panic!("an exhausted lock pool must surface an error, not a guard"), + Err(e) => e, + }; + assert!( + started.elapsed() < Duration::from_secs(6), + "the error must arrive on the acquire timeout, not after a long hang" + ); + assert!( + err.to_string().contains("lock-pool connection"), + "the error must name the lock-pool checkout, got: {err}" + ); + + held.release(false).await; + } + + /// Round trip: the lock is observably HELD between acquire and release, and + /// observably FREE after. Both checks run from an independent session; from + /// the holding session they would pass vacuously (session locks are + /// re-entrant) and would not notice an unlock that landed on the wrong + /// connection. + #[sqlx::test] + async fn acquire_write_holds_the_lock_until_release(pool: PgPool) { + let probe = sibling_pool(&pool, 2); + let owner = "did:key:z6MkRoundTrip"; + let repo = "round-trip"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + + let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-test-repos"), pool.clone()); + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + assert!( + !lock_is_free_elsewhere(&probe, key).await, + "the lock must be held while the guard is alive" + ); + + // No polling here, deliberately. `release` must free the lock SYNCHRONOUSLY, + // which it can only do by unlocking on the connection that took it; a + // `pg_advisory_unlock` sent through the pool would land on some other + // session and return false. The `after_release` hook is a net for the + // cancellation path and fires from a spawned task well after this point, so + // it must not be what makes this assertion pass. + guard.release(true).await; + assert!( + lock_is_free_elsewhere(&probe, key).await, + "release must free the lock as seen from another session" + ); + } + + /// `release(false)` skips the Tigris upload but must still free the lock; a + /// failed write that kept the lock would wedge the repo. + #[sqlx::test] + async fn release_after_failed_write_still_frees_the_lock(pool: PgPool) { + let probe = sibling_pool(&pool, 2); + let owner = "did:key:z6MkFailedWrite"; + let repo = "failed-write"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + + let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-test-repos"), pool.clone()); + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + guard.release(false).await; + + assert!( + wait_until_free(&probe, key, Duration::from_secs(5)).await, + "release(success = false) must still free the lock" + ); + } + + /// The lock is per repo: a second acquire for the SAME repo waits for the + /// first to release, while a different repo proceeds straight through. + #[sqlx::test] + async fn same_repo_acquires_serialize_and_different_repos_do_not(pool: PgPool) { + let repos_dir = PathBuf::from("/tmp/gitlawb-test-repos"); + let owner = "did:key:z6MkSerialize"; + let store = RepoStore::for_testing(repos_dir, pool.clone()); + + let first = store + .acquire_write(owner, "serialize-a") + .await + .expect("first acquire"); + + // Different repo: unaffected by the held lock. + let other = tokio::time::timeout( + Duration::from_secs(2), + store.acquire_write(owner, "serialize-b"), + ) + .await + .expect("a different repo must not wait on this lock") + .expect("acquire other repo"); + other.release(false).await; + + // Same repo: must not acquire while `first` is alive. + let contender = tokio::spawn({ + let store = store.clone(); + async move { store.acquire_write(owner, "serialize-a").await } + }); + tokio::time::sleep(Duration::from_millis(1500)).await; + assert!( + !contender.is_finished(), + "a second acquire for the same repo must block while the first guard lives" + ); + + first.release(false).await; + let second = tokio::time::timeout(Duration::from_secs(10), contender) + .await + .expect("contender must finish once the lock is free") + .expect("contender task") + .expect("contender acquire"); + second.release(false).await; + } // ── repo_name validation ─────────────────────────────────────────────── diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 1ed45929..fe18c706 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -279,8 +279,20 @@ async fn main() -> Result<()> { None }; - let repo_store = - git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, db.pool().clone()); + // Repo write locks run on their own pool, never the main query pool: each + // push holds its connection for the whole receive-pack, and + // db_max_connections (20) is below max_concurrent_git_pushes (32), so sharing + // would starve every other query under a push burst. Headroom above the push + // cap keeps a push from ever queueing here for a connection where it did not + // before. See build_lock_pool for the cancellation semantics (#173). + let lock_pool = git::repo_store::build_lock_pool( + db.pool(), + u32::try_from(config.max_concurrent_git_pushes) + .unwrap_or(u32::MAX) + .saturating_add(8), + std::time::Duration::from_secs(config.db_acquire_timeout_secs), + ); + let repo_store = git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, lock_pool); // Per-DID limiter for the creation endpoints. Keyed on the authenticated // DID (attacker-varied), so bound its key set to cap memory. From 2961ba73ec4acf45a405570ef03af606b93dfbaa Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:52:44 -0500 Subject: [PATCH 17/77] fix(node): retry the requeue re-read so a coalesced push is not silently dropped `requeue_or_release` clears the dirty flag atomically with the decision to loop, which is correct and stays as it is. The problem was what the pass did next: a failed `get_repo_by_id` fell into an Err arm that yielded no rules, and `list_visibility_rules(...).ok()` collapsed its own error the same way. With no rules, `replication_withheld_set` returns `(false, None)`, the pass skipped its work entirely, and the loop exited having consumed the dirty bit that stood for the coalesced push. There is no reconciliation sweep to re-derive it, so that push's objects were never pinned and its withheld blobs never sealed. `requeue_refresh_state` now separates the three cases the old code collapsed. `Ok(Some)` proceeds. `Ok(None)` means the repo really is gone and releases without touching the retry budget. `Err` from either read is transient and is retried with doubling backoff up to three attempts. A rules-read error is no longer indistinguishable from a repo that has no rules. Exhausting the retries still drops the pass, which is the pre-existing residual, but it now logs at error level with the repo id and attempt count instead of disappearing behind a warning. Six tests cover the retried-then-lands path, the bounded-exhaustion path, repo-gone, the rules-read arm, freshness of the applied rules, and the coalescing property that had to survive. Reverting the Err arm, restoring the `.ok()`, raising the bound, or downgrading the error log each turns a test red. --- crates/gitlawb-node/src/api/repos.rs | 198 +++++++++- crates/gitlawb-node/src/test_support.rs | 501 ++++++++++++++++++++++++ 2 files changed, 685 insertions(+), 14 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b817318d..35c49dd8 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -849,6 +849,177 @@ async fn requeue_full_scan_object_list( .await } +/// Test-only fault-injection seam for the requeue re-read (#173 U2). The defect this +/// unit fixes lives entirely on the `Err` arm of the two re-reads, which a real Postgres +/// pool will not produce on demand, so the two reads go through the wrappers below and +/// consult this table first. Keyed by `repo_id` (a fresh uuid per test) so tests running +/// in parallel in one process cannot see each other's injections, and it also records +/// the ATTEMPT counts the retry-bound assertions key on. +#[cfg(test)] +pub(crate) mod requeue_faults { + use std::collections::HashMap; + use std::sync::{Mutex, OnceLock}; + + #[derive(Default, Clone, Copy, Debug)] + pub(crate) struct Counters { + pub(crate) repo_read_failures_left: usize, + pub(crate) rules_read_failures_left: usize, + pub(crate) repo_read_attempts: usize, + pub(crate) rules_read_attempts: usize, + } + + fn table() -> &'static Mutex> { + static TABLE: OnceLock>> = OnceLock::new(); + TABLE.get_or_init(|| Mutex::new(HashMap::new())) + } + + /// Make the next `repo_read_failures` repo re-reads and the next + /// `rules_read_failures` rule re-reads for `repo_id` return `Err`, then succeed. + pub(crate) fn inject(repo_id: &str, repo_read_failures: usize, rules_read_failures: usize) { + table().lock().unwrap().insert( + repo_id.to_string(), + Counters { + repo_read_failures_left: repo_read_failures, + rules_read_failures_left: rules_read_failures, + ..Default::default() + }, + ); + } + + /// Observed attempt counts (and remaining injections) for `repo_id`. + pub(crate) fn counters(repo_id: &str) -> Counters { + table() + .lock() + .unwrap() + .get(repo_id) + .copied() + .unwrap_or_default() + } + + /// Production-path hook: count one repo re-read attempt, return whether it must fail. + pub(crate) fn take_repo_read(repo_id: &str) -> bool { + let mut map = table().lock().unwrap(); + let c = map.entry(repo_id.to_string()).or_default(); + c.repo_read_attempts += 1; + if c.repo_read_failures_left > 0 { + c.repo_read_failures_left -= 1; + return true; + } + false + } + + /// Production-path hook: count one rules re-read attempt, return whether it must fail. + pub(crate) fn take_rules_read(repo_id: &str) -> bool { + let mut map = table().lock().unwrap(); + let c = map.entry(repo_id.to_string()).or_default(); + c.rules_read_attempts += 1; + if c.rules_read_failures_left > 0 { + c.rules_read_failures_left -= 1; + return true; + } + false + } +} + +/// Attempts allowed for the requeue re-read before the task gives up (#173 U2). The +/// dirty flag that represented the coalesced push is already consumed by the atomic +/// check-and-clear at the tail and `EncryptInflightGuard::drop` removes the key, so the +/// flag cannot outlive the task and there is no reconciliation sweep to re-derive the +/// work: a transient read error must be RETRIED here or the push's pin/encrypt pass is +/// gone. The bound keeps a sustained outage from spinning forever; on exhaustion the +/// work is still lost (the pre-existing residual), but the give-up is logged at ERROR +/// so it is observable instead of silent. +const REQUEUE_REREAD_MAX_ATTEMPTS: usize = 3; + +/// Backoff before the next re-read attempt. Doubles per attempt. +const REQUEUE_REREAD_BACKOFF: std::time::Duration = std::time::Duration::from_millis(50); + +/// The outcome of the requeue's fresh state re-read, keeping the three cases the old +/// code collapsed into one distinct: a usable refresh, a repo that genuinely no longer +/// exists (terminal, and NOT a retry), and a transient read failure (retryable). +enum RequeueRefresh { + State { + rules: Vec, + is_public: bool, + owner_did: String, + }, + Gone, + Failed, +} + +/// Re-read repo state for a requeue pass, retrying transient read errors. +/// +/// Both reads are retryable and neither may be read as an absence: an `Err` from the +/// repo row is not "the repo is gone", and an `Err` from the rule list is not "this repo +/// has no rules" (the old `.ok()` made those indistinguishable, and a `None` rule set +/// makes `replication_withheld_set` return `None`, which skips the entire pass). Only +/// `Ok(None)` on the repo row is a terminal absence, and it consumes no retry budget. +async fn requeue_refresh_state(ctx: &PostPushReplication) -> RequeueRefresh { + let mut backoff = REQUEUE_REREAD_BACKOFF; + for attempt in 1..=REQUEUE_REREAD_MAX_ATTEMPTS { + let record = match requeue_get_repo(ctx).await { + Ok(Some(rec)) => rec, + Ok(None) => { + tracing::debug!(repo = %ctx.repo_id, "repo gone before requeue pass; releasing"); + return RequeueRefresh::Gone; + } + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, err = %e, attempt, + "requeue repo re-read failed; retrying" + ); + tokio::time::sleep(backoff).await; + backoff *= 2; + continue; + } + }; + match requeue_list_rules(ctx).await { + Ok(rules) => { + return RequeueRefresh::State { + rules, + is_public: record.is_public, + owner_did: record.owner_did, + } + } + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, err = %e, attempt, + "requeue visibility-rule re-read failed; retrying" + ); + tokio::time::sleep(backoff).await; + backoff *= 2; + } + } + } + tracing::error!( + repo = %ctx.repo_id, + attempts = REQUEUE_REREAD_MAX_ATTEMPTS, + "requeue re-read failed on every attempt; the coalesced push's pin/encrypt pass is \ + dropped (no reconciliation sweep re-derives it)" + ); + RequeueRefresh::Failed +} + +/// The requeue's repo re-read, behind the test-only fault seam above. +async fn requeue_get_repo(ctx: &PostPushReplication) -> anyhow::Result> { + #[cfg(test)] + if requeue_faults::take_repo_read(&ctx.repo_id) { + return Err(anyhow::anyhow!("injected repo re-read failure")); + } + ctx.db.get_repo_by_id(&ctx.repo_id).await +} + +/// The requeue's visibility-rule re-read, behind the test-only fault seam above. +async fn requeue_list_rules( + ctx: &PostPushReplication, +) -> anyhow::Result> { + #[cfg(test)] + if requeue_faults::take_rules_read(&ctx.repo_id) { + return Err(anyhow::anyhow!("injected visibility-rule re-read failure")); + } + ctx.db.list_visibility_rules(&ctx.repo_id).await +} + /// The detached post-push encryption + local-IPFS pin task, as a REQUEUE LOOP. /// /// Pass one uses the spawn-time captures (`first_*`) — the delta the push handler @@ -966,20 +1137,19 @@ async fn run_post_push_replication( // A push coalesced during this pass. Re-read repo state FRESH (never the stale // spawn-time captures) so a coalesced push that changed `.gitlawb` withholding // is walked under the new policy, then re-enumerate the pin set fail-closed. - let (r_rules, r_is_public, r_owner) = match ctx.db.get_repo_by_id(&ctx.repo_id).await { - Ok(Some(rec)) => ( - ctx.db.list_visibility_rules(&ctx.repo_id).await.ok(), - rec.is_public, - rec.owner_did, - ), - Ok(None) => { - tracing::debug!(repo = %ctx.repo_id, "repo gone before requeue pass; releasing"); - (None, false, String::new()) - } - Err(e) => { - tracing::warn!(repo = %ctx.repo_id, err = %e, "requeue repo re-read failed; skipping this pass's work"); - (None, false, String::new()) - } + // A read error here is retried rather than treated as "no state" (#173 U2): the + // dirty flag is already consumed, so skipping the pass would silently discard + // exactly the push this requeue exists to cover. + let (r_rules, r_is_public, r_owner) = match requeue_refresh_state(&ctx).await { + RequeueRefresh::State { + rules, + is_public, + owner_did, + } => (Some(rules), is_public, owner_did), + // The repo is gone (terminal) or the re-read never succeeded (already logged + // at ERROR). Either way there is no fresh state to act on, so exit; the guard + // Drop removes the key so the repo is never locked out of a future task. + RequeueRefresh::Gone | RequeueRefresh::Failed => break, }; let (_announce, r_withheld) = replication_withheld_set( r_rules.clone(), diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 0d0ae2e8..f97981b4 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -8323,5 +8323,506 @@ mod tests { "the key is released after one pass" ); } + + // ---- U2 (#173): a transient re-read failure must not discard the coalesced + // push's work ---- + // + // The requeue pass re-reads repo state fresh. Before this unit, the `Err` arm of + // either read (repo row, visibility rules) collapsed into "no rules", which made + // `replication_withheld_set` return `None`, which skipped the whole pass. The + // dirty flag was already consumed by the atomic check-and-clear at the tail and + // there is no reconciliation sweep, so the coalesced push's pin/encrypt work was + // lost silently. These tests drive that `Err` arm through the fault seam in + // `api::repos::requeue_faults` (a real pool will not fail on demand) and assert + // on the WORK PERFORMED, not on control flow. + mod u2_reread_retry { + use super::*; + use crate::api::repos::requeue_faults; + + /// Process-wide tracing capture so a test can assert the give-up is logged at + /// ERROR. A global default subscriber can only be installed once per process, + /// so it is shared by every test here and assertions filter on the repo id, + /// which is a fresh uuid per test. + mod logcap { + use std::sync::{Arc, Mutex, OnceLock}; + use tracing::{Event, Level, Subscriber}; + use tracing_subscriber::layer::{Context, Layer}; + use tracing_subscriber::prelude::*; + + type Lines = Arc>>; + + fn lines() -> &'static Lines { + static LINES: OnceLock = OnceLock::new(); + LINES.get_or_init(|| Arc::new(Mutex::new(Vec::new()))) + } + + struct Capture; + impl Layer for Capture { + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + struct V(String); + impl tracing::field::Visit for V { + fn record_debug( + &mut self, + field: &tracing::field::Field, + value: &dyn std::fmt::Debug, + ) { + self.0.push_str(&format!(" {}={:?}", field.name(), value)); + } + } + let mut v = V(String::new()); + event.record(&mut v); + lines() + .lock() + .unwrap() + .push((*event.metadata().level(), v.0)); + } + } + + pub(super) fn install() { + static ONCE: OnceLock<()> = OnceLock::new(); + ONCE.get_or_init(|| { + let _ = tracing::subscriber::set_global_default( + tracing_subscriber::registry().with(Capture), + ); + }); + } + + pub(super) fn errors_containing(needle: &str) -> Vec { + lines() + .lock() + .unwrap() + .iter() + .filter(|(lvl, msg)| *lvl == Level::ERROR && msg.contains(needle)) + .map(|(_, msg)| msg.clone()) + .collect() + } + } + + /// SCENARIO 1. The repo re-read fails once, then succeeds: the requeue pass + /// must still RUN, under the refreshed state, and pin the coalesced push's + /// object. RED before the fix (the single `Err` yielded `(None, false, "")`, + /// the pass was skipped, and the already-consumed dirty flag meant the work + /// was gone for good). + #[sqlx::test] + async fn u2_transient_repo_reread_failure_is_retried_and_work_lands(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-retry"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + // The coalesced push B adds obj2, absent from push A's spawn captures. + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // One transient repo re-read failure, then the real DB answers. + requeue_faults::inject(&repo.id, 1, 0); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces while A is in flight" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj1.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's object is pinned after the retried re-read (RED \ + before this unit: the Err arm dropped the pass and the work with it)" + ); + let c = requeue_faults::counters(&repo.id); + assert_eq!( + c.repo_read_attempts, 2, + "the failed re-read is retried exactly once before it succeeds" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits" + ); + } + + /// SCENARIO 2. Every re-read attempt fails: the loop must give up on a BOUND + /// (asserted as a literal, so raising or removing the bound goes RED) and log + /// the give-up at ERROR so the residual loss is observable rather than silent. + #[sqlx::test] + async fn u2_sustained_repo_reread_failure_is_bounded_and_logged(pool: PgPool) { + logcap::install(); + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-bounded"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Far more failures than the bound allows: the outage never clears. + requeue_faults::inject(&repo.id, 10_000, 0); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj1.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + let c = requeue_faults::counters(&repo.id); + assert_eq!( + c.repo_read_attempts, 3, + "the re-read is bounded at 3 attempts; unbounded retry or a raised \ + bound must fail here" + ); + assert!( + !state.db.is_pinned(&obj2).await.unwrap(), + "with the read never succeeding there is nothing fresh to act on" + ); + let errs = logcap::errors_containing(&repo.id); + assert!( + !errs.is_empty(), + "the exhausted requeue re-read is logged at ERROR with the repo id, so \ + the residual work loss is observable; captured: {errs:?}" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is still released on the give-up path" + ); + } + + /// SCENARIO 3. `Ok(None)` (the repo was deleted during the in-flight window) + /// is NOT a transient failure: it must release immediately without burning the + /// retry budget. The repo id is never inserted, so the re-read legitimately + /// returns `Ok(None)`. + #[sqlx::test] + async fn u2_repo_gone_releases_without_consuming_retries(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let missing_id = uuid::Uuid::new_v4().to_string(); + let git_repo = init_repo(); + let _obj1 = commit(&git_repo.path, "a.txt", "one\n"); + + let server = mockito::Server::new_async().await; + + requeue_faults::inject(&missing_id, 0, 0); + + let guard = state + .encrypt_inflight + .try_begin(&missing_id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&missing_id).is_none(), + "push B coalesces" + ); + + // Empty object list: pass one touches no pin rows for a repo that is gone. + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + missing_id.clone(), + server.url(), + true, + owner.clone(), + vec![], + Some(vec![]), + HashSet::new(), + ) + .await; + + let c = requeue_faults::counters(&missing_id); + assert_eq!( + c.repo_read_attempts, 1, + "a deleted repo is a terminal answer, never retried" + ); + assert_eq!( + c.rules_read_attempts, 0, + "no rules read is attempted once the repo row is gone" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released cleanly" + ); + } + + /// SCENARIO 4. A failed visibility-rule read is transient, never an empty + /// policy. RED before the fix, where `.ok()` made "the rules read failed" and + /// "this repo has no rules" the same value: the withheld blob was then neither + /// sealed nor covered, because a `None` rule set skips the pass entirely. + #[sqlx::test] + async fn u2_transient_rules_read_failure_is_retried_not_read_as_empty(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u2-rules"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + + // The coalesced push B is what added the path-scoped rule. + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + &[reader], + &owner, + ) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // The repo row reads fine; the RULES read is the one that blips. + requeue_faults::inject(&repo.id, 0, 1); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + // Push A's captures are stale: no rule, nothing withheld. + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![pub_oid.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "the withheld blob is sealed under the RETRIED rule set (RED with \ + list_visibility_rules(..).ok(): an empty policy seals nothing)" + ); + let c = requeue_faults::counters(&repo.id); + assert_eq!( + c.rules_read_attempts, 2, + "the failed rules read is retried, not collapsed into an empty rule set" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "the withheld blob is never pinned in the clear by the requeue" + ); + } + + /// SCENARIO 5. The fault-free control for scenario 4: the rules applied by the + /// requeue are the COALESCED push's fresh ones, never the spawn-time capture, + /// and the retry path does not perturb that (exactly one read of each). + #[sqlx::test] + async fn u2_requeue_applies_fresh_rules_not_spawn_captures(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u2-fresh"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + &[reader], + &owner, + ) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + requeue_faults::inject(&repo.id, 0, 0); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![pub_oid.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + let c = requeue_faults::counters(&repo.id); + assert_eq!( + (c.repo_read_attempts, c.rules_read_attempts), + (1, 1), + "a healthy DB is read exactly once per requeue pass" + ); + assert!( + state.db.is_pinned(&pub_oid).await.unwrap(), + "the visible object is pinned under the fresh rules" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "the freshly-read rule withholds the secret blob (the spawn-time \ + capture had no rules at all)" + ); + } + + /// SCENARIO 6. Regression guard on the property the fix must not disturb: the + /// tail check-and-clear is atomic, so a push coalescing during it is still + /// covered by exactly one more pass, and the key is released after. + #[sqlx::test] + async fn u2_coalesced_push_still_covered_by_exactly_one_requeue_pass(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-coalesce"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + requeue_faults::inject(&repo.id, 0, 0); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + // Push B lands during the in-flight window: dirty flag set. + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + assert_eq!( + state.encrypt_inflight.dirty(&repo.id), + Some(true), + "the coalesced push marked the repo dirty" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj1.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert_eq!( + requeue_faults::counters(&repo.id).repo_read_attempts, + 1, + "one coalesced push means exactly one requeue pass, no re-spin" + ); + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "push A's object is pinned" + ); + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's object is covered by the requeue pass" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the key is released once the task is clean" + ); + } + } } } From ff189ceab92671de3ec245ccd2fb59141e23e2f5 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:40:03 -0500 Subject: [PATCH 18/77] fix(node): record a failed pin-source write so the resolver keeps its fallback The resolver treated a non-empty, below-cap pin-source set as proof that every source had been recorded, and skipped the bounded scan on that basis. But `record_pin_source` is best effort at every call site: the ipfs_pin sites were warn-only once their retries exhausted, and both pinata sites had no retry at all. An object first pinned from a private repo and later pushed from a public one whose source write failed keeps a set naming only the private source, so `GET /ipfs/{cid}` returned 404 for an object the public repo would have served. Migration v15 adds `pinned_cids.pin_sources_incomplete`, NOT NULL DEFAULT FALSE so every existing row reads as complete. A record that fails outright sets it; a later successful record clears it, inside the same transaction as the insert so the two paths cannot drift. `needs_scan` now ORs the marker in, short-circuiting after the existing empty and at-cap checks so it costs nothing on the serve path. Pinata's two bare calls now share `retry_db_record` with ipfs_pin. The ipfs_pin first-pin path additionally records the CID and its source in one transaction; Pinata's remains two retried calls. INV-10 is preserved: a complete set still tail-404s with no preload, asserted on the existing preload counter. The fallback is not an authorization bypass, and the test for that asserts the scan actually ran before checking the denial, so the denial cannot pass vacuously. Dropping the marker term from the gate, forcing it true, or removing the clear each turns a test red. The marker is per-object rather than per-(object, repo): a successful record from one repo clears a marker set by another's failure. That is the conservative direction, since the marker only ever adds the fallback back. --- crates/gitlawb-node/src/api/ipfs.rs | 24 +- crates/gitlawb-node/src/db/mod.rs | 119 +++++- crates/gitlawb-node/src/ipfs_pin.rs | 36 +- crates/gitlawb-node/src/pinata.rs | 31 +- crates/gitlawb-node/src/test_support.rs | 542 ++++++++++++++++++++++++ 5 files changed, 728 insertions(+), 24 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 624a1b27..ba669027 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -329,15 +329,31 @@ pub async fn get_by_cid( // (e.g. a later PUBLIC pinner buried by 16 attacker sources — the // pin-source griefing hole). The scan gates every repo through the // real per-caller gate, so it finds that copy. - // A non-empty, non-full set is COMPLETE (every recorded source was just tried), so - // skip the scan and let the tail 404 — ordinary denials never fan out to O(repos) - // (INV-10 / F3). The at_cap query runs only on a provenance MISS (we return above - // on Served), so it never costs the serve path. + // - marked -> a `record_pin_source` for this object failed outright (U3, #173). + // `record_pin_source` is best effort at every pin call site, so a + // non-empty below-cap set is NOT self-evidently complete: an object + // first pinned from a PRIVATE repo and later pushed from a PUBLIC + // one whose record failed names only the private source. The + // durable `pin_sources_incomplete` marker is the node's own record + // that a source is missing, so the fallback stays available for + // exactly those objects instead of 404ing a servable public copy. + // Only a set with NONE of these three signals is treated as complete (every + // recorded source was just tried), so it skips the scan and lets the tail 404, and + // ordinary denials never fan out to O(repos) (INV-10 / F3). Both extra queries run + // only on a provenance MISS (we return above on Served), so neither costs the serve + // path, and the fallback is not an authorization bypass: the scan gates every repo + // through the SAME per-caller gate, so a caller who may not read the object is + // still denied. let needs_scan = sources.is_empty() || state .db .pin_sources_at_cap(sha256_hex) .await + .map_err(AppError::Internal)? + || state + .db + .pin_sources_incomplete(sha256_hex) + .await .map_err(AppError::Internal)?; if needs_scan { // F3 (#173, INV-10/INV-15): peek the per-IP WORK-budget limiter WITHOUT diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 775e7c17..c2a6cb60 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -937,6 +937,25 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS legacy_provider_cid TEXT", ], }, + Migration { + version: 15, + name: "pinned_cids_sources_incomplete", + stmts: &[ + // U3 (#173): `record_pin_source` is BEST EFFORT at every call site, so a + // non-empty, below-cap source set is not proof that every source was + // recorded. An object first pinned from a private repo and later pushed + // from a PUBLIC repo whose record failed keeps a set naming only the + // private source, and the resolver used to call that set complete and 404 + // an object the public repo would serve. Record the miss DURABLY here so + // `GET /ipfs/{cid}` keeps the bounded scan fallback for exactly those + // objects. Not inferable from row counts or timestamps: neither can tell + // "no other source exists" from "a source failed to record", which is the + // whole distinction. NEW versioned migration (never appended to an applied + // block, INV-7). NOT NULL DEFAULT FALSE so every pre-existing row reads as + // complete and ordinary denials stay off the O(repos) path (INV-10). + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS pin_sources_incomplete BOOLEAN NOT NULL DEFAULT FALSE", + ], + }, ]; /// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). @@ -2282,6 +2301,11 @@ impl Db { /// gets it filled the next time the object is re-pinned with a known source. /// `cid`/`pinned_at` are left untouched on conflict. `repo_id` is `None` only /// for a legacy pin with no known source; those fall back to the resolver's scan. + /// + /// The production first-pin path now goes through [`Self::record_pinned_cid_with_source`] + /// (U3, #173) so the pin and its source land atomically; this remains the seam for + /// seeding legacy, source-less rows in tests. + #[cfg_attr(not(test), allow(dead_code))] pub async fn record_pinned_cid( &self, sha256_hex: &str, @@ -2384,7 +2408,17 @@ impl Db { /// (`pin_sources_for_oid`) caps the ADDITIONAL sources at `MAX_PIN_SOURCES` (always /// keeping the first-pinner), so the INV-10 bound on serve-time work holds at /// `O(MAX_PIN_SOURCES + 1)` regardless of a table overshoot. + /// + /// A successful record also CLEARS the `pin_sources_incomplete` marker for the + /// object, in the SAME transaction as the insert (U3, #173), so the clear cannot + /// drift across the four call sites or land without the row it describes. The + /// marker is per-object, not per-(object, repo): a record from repo B clears a + /// marker set by a failed record from repo A, which can re-hide A's hole until A + /// pushes again. That is the deliberate cost of a single boolean, and it fails in + /// the safe direction relative to today (the marker only ever ADDS the fallback, + /// never removes a source the resolver already tries). pub async fn record_pin_source(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + let mut tx = self.pool.begin().await?; sqlx::query( "INSERT INTO pin_repo_sources (sha256_hex, repo_id) SELECT $1, $2 @@ -2394,11 +2428,94 @@ impl Db { .bind(sha256_hex) .bind(repo_id) .bind(MAX_PIN_SOURCES) - .execute(&self.pool) + .execute(&mut *tx) .await?; + sqlx::query( + "UPDATE pinned_cids SET pin_sources_incomplete = FALSE + WHERE sha256_hex = $1 AND pin_sources_incomplete", + ) + .bind(sha256_hex) + .execute(&mut *tx) + .await?; + tx.commit().await?; Ok(()) } + /// Record a first pin and its source ATOMICALLY (U3, #173). The first-pin path + /// used to run `record_pinned_cid` and `record_pin_source` as two independent + /// best-effort calls, so the pin could land while its source did not, leaving a + /// source set that is silently missing its own first pinner. One transaction + /// removes that window entirely: either both rows land or neither does, and a + /// total failure leaves the object unpinned so the next push retries it. + pub async fn record_pinned_cid_with_source( + &self, + sha256_hex: &str, + cid: &str, + repo_id: &str, + ) -> Result<()> { + let mut tx = self.pool.begin().await?; + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT(sha256_hex) DO UPDATE SET + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", + ) + .bind(sha256_hex) + .bind(cid) + .bind(Utc::now().to_rfc3339()) + .bind(repo_id) + .execute(&mut *tx) + .await?; + sqlx::query( + "INSERT INTO pin_repo_sources (sha256_hex, repo_id) + SELECT $1, $2 + WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo_id) + .bind(MAX_PIN_SOURCES) + .execute(&mut *tx) + .await?; + sqlx::query( + "UPDATE pinned_cids SET pin_sources_incomplete = FALSE + WHERE sha256_hex = $1 AND pin_sources_incomplete", + ) + .bind(sha256_hex) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Mark this object's pin-source set as KNOWN INCOMPLETE (U3, #173). Called when a + /// `record_pin_source` exhausts its retries, which is the only moment the node + /// knows a source it meant to record is missing. `GET /ipfs/{cid}` reads it to keep + /// the bounded scan fallback for that object, so a public copy that would serve is + /// no longer 404'd. A no-op when no `pinned_cids` row exists (the first-pin path is + /// transactional, so there is no half-recorded pin to describe). + pub async fn mark_pin_sources_incomplete(&self, sha256_hex: &str) -> Result<()> { + sqlx::query("UPDATE pinned_cids SET pin_sources_incomplete = TRUE WHERE sha256_hex = $1") + .bind(sha256_hex) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Whether this object's pin-source set is KNOWN INCOMPLETE (U3, #173): a + /// `record_pin_source` for it failed outright and no later record has repaired the + /// set. `false` for an unpinned oid and for every row predating the column, so the + /// common path is unchanged and an ordinary denial never fans out (INV-10). + pub async fn pin_sources_incomplete(&self, sha256_hex: &str) -> Result { + let flag: Option = sqlx::query_scalar( + "SELECT pin_sources_incomplete FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(flag.unwrap_or(false)) + } + /// Every source repository recorded for a pinned object (F1, #173 jatmn round 8): /// the union of the first-pinner `pinned_cids.repo_id` and the `pin_repo_sources` /// rows, deduped and ordered for a deterministic resolver walk. diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 3d2f13df..0e58e27e 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -22,13 +22,12 @@ const PIN_RECORD_BACKOFF: Duration = Duration::from_millis(50); /// so a dropped `record_pin_source`/`record_pinned_cid` makes `GET /ipfs/{cid}` /// 404 a valid public copy. Every wrapped insert is idempotent (`ON CONFLICT DO /// NOTHING` / provenance-preserving upsert), so re-running is safe. On exhausted -/// attempts the last error is returned and the caller keeps its warn — behavior -/// degrades to the pre-retry state, not worse. Process death mid-retry or a DB -/// outage outlasting the backoff horizon leaves the same residual hole (no -/// persisted marker to reconcile from at startup), retired only by a future -/// reconciliation sweep. Runs inside the already-detached post-push task, so the -/// backoff adds no push latency. -async fn retry_db_record(mut op: F) -> Result<()> +/// attempts the last error is returned and the caller records the durable +/// `pin_sources_incomplete` marker (U3, #173), which is what keeps the resolver's +/// bounded scan fallback available for that object instead of 404ing a public copy. +/// Shared with the `pinata.rs` twin so both pin paths retry identically. Runs +/// inside the already-detached post-push task, so the backoff adds no push latency. +pub(crate) async fn retry_db_record(mut op: F) -> Result<()> where F: FnMut() -> Fut, Fut: std::future::Future>, @@ -254,6 +253,14 @@ pub async fn pin_new_objects( // when this repo would serve it. Bounded per object (MAX_PIN_SOURCES). if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + // U3 (#173): the retries are spent and this repo is NOT in the source + // set, so the set is known incomplete. Persist that, or the resolver + // reads a non-empty below-cap set as COMPLETE and 404s an object this + // repo would serve. Warn-only in turn: if the marker write also fails + // the object degrades to the pre-U3 behavior, never worse. + if let Err(e) = db.mark_pin_sources_incomplete(&sha).await { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } } // R8 (#173 round 10): opportunistically repair a legacy provider-CID // row (Kubo dag-pb / Pinata) to the raw-content resolver key on this @@ -300,16 +307,19 @@ pub async fn pin_new_objects( // verifies them against the requested CID, so the raw CID is the correct // key. Mirrors the pinata twin, which already records the raw CID. let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); + // F1 (#173 round 8): the first pinner is recorded in pin_repo_sources too, + // so every source (first and subsequent) is tried uniformly by the + // resolver. U3 (#173): the pin and its source go down in ONE transaction. + // As two independent best-effort calls this path could land the pin while + // dropping its own source, producing a source set silently missing its + // first pinner; atomically there is no such window, and a total failure + // leaves the object unpinned so the next push retries the whole thing. if let Err(e) = - retry_db_record(|| db.record_pinned_cid(&sha, &raw_cid, Some(repo_id))).await + retry_db_record(|| db.record_pinned_cid_with_source(&sha, &raw_cid, repo_id)) + .await { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); } - // F1 (#173 round 8): also record the first pinner in pin_repo_sources so - // every source (first and subsequent) is tried uniformly by the resolver. - if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { - tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); - } // Return the provider Hash (not the resolver key), mirroring the pinata // twin's contract: the DB `cid` is the raw resolver key (recorded above), // the returned value is the provider CID. Here the return is consumed only diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 4d6b9704..e49f9205 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -114,9 +114,17 @@ pub async fn pin_new_objects( } // F1 (#173 round 8): record this repo as an additional source for the // already-pinned object (mirrors the ipfs_pin skip-branch insert) so the - // resolver can serve a shared object from any pin-path source. - if let Err(e) = db.record_pin_source(&sha, repo_id).await { + // resolver can serve a shared object from any pin-path source. U3 (#173): + // retried through the SHARED helper (this was a bare call, so a single + // transient error dropped the source outright) and, on exhaustion, marked + // durably so the resolver keeps the bounded scan fallback for the object. + if let Err(e) = + crate::ipfs_pin::retry_db_record(|| db.record_pin_source(&sha, repo_id)).await + { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + if let Err(e) = db.mark_pin_sources_incomplete(&sha).await { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } } continue; } @@ -143,15 +151,26 @@ pub async fn pin_new_objects( // dag-pb/UnixFS, so its returned CID does not hash the raw content and // must not become an alias `/ipfs/{cid}` serves raw git bytes for (#173). let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); - if let Err(e) = db - .record_pinata_cid(&sha, &raw_cid, &cid, Some(repo_id)) - .await + // U3 (#173): both records go through the shared retry helper, at parity + // with the ipfs_pin twin. These were bare calls, so one transient DB error + // permanently dropped a pin source. + if let Err(e) = crate::ipfs_pin::retry_db_record(|| { + db.record_pinata_cid(&sha, &raw_cid, &cid, Some(repo_id)) + }) + .await { tracing::warn!(sha = %sha, err = %e, "failed to record pinata_cid in DB"); } // F1 (#173 round 8): also record the first pinner in pin_repo_sources. - if let Err(e) = db.record_pin_source(&sha, repo_id).await { + // U3: an exhausted retry marks the set incomplete so the resolver keeps + // the scan fallback rather than 404ing a copy it could serve. + if let Err(e) = + crate::ipfs_pin::retry_db_record(|| db.record_pin_source(&sha, repo_id)).await + { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + if let Err(e) = db.mark_pin_sources_incomplete(&sha).await { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } } pinned.push((sha, cid)); } diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index f97981b4..fd90e46a 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -2690,6 +2690,548 @@ mod tests { ); } + // ── U3 (#173): durable pin-source incompleteness marker ────────────────── + // + // `record_pin_source` is best effort at every call site, so a non-empty, + // below-cap source set is NOT proof of completeness: an object first pinned + // from a PRIVATE repo and later pushed from a PUBLIC repo whose record failed + // has a set that names only the private source. The resolver used to treat + // that set as complete and 404 an object the public repo would serve. The + // pinned_cids.pin_sources_incomplete marker records the miss durably so the + // bounded scan fallback still runs. These tests drive both arms: the marker + // set (fallback runs, object serves, denial still denies) and the marker + // clear (ordinary denials stay off the O(repos) path, INV-10). + + /// Make `record_pin_source` fail for the duration of `body` by moving the + /// `pin_repo_sources` table out from under it, the closest honest stand-in for + /// the transient DB error the retry wrapper is there to absorb. Every other + /// pin-path query keeps working, so only the source record (and its retries) + /// fails, which is exactly the partial-record shape the finding turns on. + async fn with_pin_sources_broken(pool: &PgPool, body: F) -> T + where + F: FnOnce() -> Fut, + Fut: std::future::Future, + { + sqlx::query("ALTER TABLE pin_repo_sources RENAME TO pin_repo_sources_hidden") + .execute(pool) + .await + .expect("hide pin_repo_sources"); + let out = body().await; + sqlx::query("ALTER TABLE pin_repo_sources_hidden RENAME TO pin_repo_sources") + .execute(pool) + .await + .expect("restore pin_repo_sources"); + out + } + + /// Pin `oid` from `repo_id` through the real ipfs_pin path with a mock Kubo that + /// must NOT be called (the object is already pinned, so this drives the + /// skip-branch `record_pin_source` and nothing else). + async fn repin_via_skip_branch( + state: &AppState, + bare: &std::path::Path, + oid: &str, + repo_id: &str, + ) { + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![oid.to_string()], + &state.db, + repo_id, + ) + .await; + m.assert_async().await; + } + + /// U3 scenario 1 (#173, the finding's exact case): an object first pinned from a + /// PRIVATE repo, then pushed from a PUBLIC repo whose `record_pin_source` + /// exhausts its retries. The source set is non-empty and below cap, so the old + /// gate called it COMPLETE and 404'd an object the public repo would happily + /// serve. With the durable marker the bounded scan fallback still runs and the + /// public copy serves. RED before the marker (404); GREEN after (200). + #[sqlx::test] + async fn ipfs_cid_incomplete_source_set_falls_back_to_scan(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3priv", "u3pub"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3priv.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3pub.git"); + + // Private first-pinner owns the only recorded source. + let mut priv_repo = seed_repo(&owner_did, "u3priv"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // The PUBLIC repo holds the same object, but its source record never lands. + let pub_repo = seed_repo(&owner_did, "u3pub"); // public, no rule + state.db.create_repo(&pub_repo).await.expect("seed public"); + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &pub_bare, &fx.public_oid, &pub_repo.id) + }) + .await; + + // The recorded set still names only the private repo, and it is below cap. + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![priv_repo.id.clone()], + "the public source really did fail to record" + ); + assert!( + !state.db.pin_sources_at_cap(&fx.public_oid).await.unwrap(), + "the set is below cap, so at_cap cannot be what triggers the fallback" + ); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a KNOWN-incomplete source set must keep the scan fallback so the public copy serves" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + } + + /// U3 scenario 2 (#173, INV-10 guard): the marker must not turn ORDINARY denials + /// into an O(repos) fan-out. With the marker false, a non-empty below-cap source + /// set and a provenance miss, the request must 404 WITHOUT the scan preload ever + /// running. The preload counter is the both-ways proof: forcing the marker true + /// unconditionally turns this red (count 1), which is what keeps the assertion + /// from being vacuous. + #[sqlx::test] + async fn ipfs_cid_complete_source_set_never_preloads(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["u3only"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3only.git"); + + // One PRIVATE source, recorded cleanly: the set is complete and below cap. + let mut priv_repo = seed_repo(&owner_did, "u3only"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &priv_repo.id).await; + state + .db + .record_pin_source(&fx.secret_oid, &priv_repo.id) + .await + .expect("record source"); + assert!( + !state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "a clean record leaves the set marked complete" + ); + assert!( + !state.db.pin_sources_at_cap(&fx.secret_oid).await.unwrap(), + "the set is below cap, so at_cap cannot be what drives the gate" + ); + + crate::api::ipfs::reset_preload_queries(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "an anonymous caller denied by the only recorded source gets the opaque 404" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld object" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 0, + "an ordinary denial against a COMPLETE source set must never run the O(repos) preload (INV-10)" + ); + } + + /// U3 scenario 3 (#173): the marker is not permanent. Once a later + /// `record_pin_source` for the object succeeds, nothing is missing, so the marker + /// clears and the scan stops being triggered. BOTH sources here are private, so the + /// provenance walk MISSES and the request actually reaches the `needs_scan` gate: + /// with a marker left stuck the gate arms the O(repos) preload for an ordinary + /// denial forever. Drop the clear and both halves go red (marker still true, preload + /// 1). A public second source would make the preload half vacuous, because the + /// provenance path serves and returns before the gate is ever evaluated. + #[sqlx::test] + async fn ipfs_cid_marker_clears_on_a_later_successful_record(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3cfirst", "u3csecond"]); + let first_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3cfirst.git"); + let second_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3csecond.git"); + + let mut first_repo = seed_repo(&owner_did, "u3cfirst"); + first_repo.is_public = false; + state.db.create_repo(&first_repo).await.expect("seed first"); + let cid = pin_cid_for_repo(&first_bare, &fx.secret_oid, &state.db, &first_repo.id).await; + let mut second_repo = seed_repo(&owner_did, "u3csecond"); + second_repo.is_public = false; + state + .db + .create_repo(&second_repo) + .await + .expect("seed second"); + + // First push from the second repo: the source record fails, so the set is marked. + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &second_bare, &fx.secret_oid, &second_repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "the exhausted record marked the set incomplete" + ); + + // A later push from the same repo records cleanly, so nothing is missing. + repin_via_skip_branch(&state, &second_bare, &fx.secret_oid, &second_repo.id).await; + assert!( + !state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "a successful record clears the marker" + ); + assert_eq!( + state + .db + .pin_sources_for_oid(&fx.secret_oid) + .await + .unwrap() + .len(), + 2, + "the repaired set really does name both sources" + ); + + crate::api::ipfs::reset_preload_queries(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "both sources are private, so the anonymous caller is denied" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld object" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 0, + "a repaired source set stops triggering the scan: the denial is back off the O(repos) path" + ); + } + + /// U3 scenario 4 (#173): the marker tracks the record's OUTCOME, not the attempt. + /// An exhausted retry sets it; a first-attempt success never does. Without the + /// second arm the first could be satisfied by marking unconditionally. + #[sqlx::test] + async fn pin_sources_incomplete_marks_only_exhausted_records(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3mark"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3mark.git"); + let repo = seed_repo(&owner_did, "u3mark"); + state.db.create_repo(&repo).await.expect("seed repo"); + let _ = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Arm A: a first-attempt success must leave the marker alone. + repin_via_skip_branch(&state, &bare, &fx.public_oid, &repo.id).await; + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "a record that lands on the first attempt never marks the set incomplete" + ); + + // Arm B: an exhausted retry marks it. + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &bare, &fx.public_oid, &repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "an exhausted record marks the set incomplete" + ); + + // An unpinned oid has no row and must read as complete, never as missing. + assert!( + !state + .db + .pin_sources_incomplete(&"f".repeat(64)) + .await + .unwrap(), + "an unpinned oid reads complete, so an unknown CID cannot arm the fallback" + ); + } + + /// U3 scenario 5 (#173): the Pinata pin path had BARE `record_pin_source` calls, so + /// one transient DB error dropped a source permanently. It now shares the ipfs_pin + /// retry helper and marks/clears the same marker. The elapsed-time assertion is the + /// retry proof: a bare call returns immediately, whereas the wrapper sleeps + /// `PIN_RECORD_BACKOFF` between each of `PIN_RECORD_ATTEMPTS` tries. + #[sqlx::test] + async fn pinata_pin_path_retries_and_marks_incomplete(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3pinata"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3pinata.git"); + let repo = seed_repo(&owner_did, "u3pinata"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // Already carries a pinata_cid, so pin_new_objects takes the skip branch and the + // only DB write under test is the source record. + let (_ty, raw) = crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .expect("object readable"); + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + state + .db + .record_pinata_cid(&fx.public_oid, &raw_cid, "QmProvider", Some(&repo.id)) + .await + .expect("seed pinata pin"); + + let client = reqwest::Client::new(); + let run = |db_broken: bool| { + let client = client.clone(); + let bare = bare.clone(); + let oid = fx.public_oid.clone(); + let repo_id = repo.id.clone(); + let state = &state; + async move { + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .with_body(r#"{"data":{"cid":"QmShouldNotHappen"}}"#) + .expect(0) + .create_async() + .await; + let started = std::time::Instant::now(); + crate::pinata::pin_new_objects( + &client, + &server.url(), + "test-jwt", + &bare, + vec![oid], + &state.db, + &repo_id, + ) + .await; + m.assert_async().await; // the upload is skipped: DB-only path + let _ = db_broken; + started.elapsed() + } + }; + + // Failing arm: retried (so it sleeps the full backoff horizon) and marked. + let elapsed = with_pin_sources_broken(&pool, || run(true)).await; + assert!( + elapsed >= std::time::Duration::from_millis(100), + "the pinata source record now RETRIES (bare call returns at once, got {elapsed:?})" + ); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "an exhausted pinata record marks the set incomplete, same as the ipfs_pin path" + ); + + // Recovery arm: a later successful pinata record clears it, same as ipfs_pin. + run(false).await; + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "a successful pinata record clears the marker" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![repo.id.clone()], + "the recovered record actually landed the source row" + ); + } + + /// U3 scenario 6 (#173, authorization): the marker arms a FALLBACK, never a bypass. + /// With the set marked incomplete and the object living only in a repo the caller + /// may not read, the scan gates every repo through the same per-caller gate, so the + /// caller is still denied and no bytes leak. + #[sqlx::test] + async fn ipfs_cid_marked_incomplete_still_denies_unauthorized_caller(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3deny"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3deny.git"); + let mut priv_repo = seed_repo(&owner_did, "u3deny"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &priv_repo.id).await; + + // The set is marked incomplete, so the fallback scan definitely runs. + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &bare, &fx.secret_oid, &priv_repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "the marker is set, so the scan fallback is armed for this object" + ); + + crate::api::ipfs::reset_preload_queries(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + crate::api::ipfs::preload_queries(), + 1, + "the fallback really did run (otherwise the denial below proves nothing)" + ); + assert_eq!( + st, + StatusCode::NOT_FOUND, + "the fallback scan gates every repo, so an unauthorized caller is still denied" + ); + assert!( + !body.contains("TOP SECRET"), + "the denial must not leak the withheld object's bytes" + ); + } + + /// U3 scenario 7 (#173, INV-7 upgrade path): a node already past v14 gets + /// `pinned_cids.pin_sources_incomplete` from the NEW v15 migration, re-running the + /// migrations is idempotent, and a row written before the column existed reads as + /// COMPLETE (so an upgrade cannot arm the O(repos) fallback for every legacy pin). + #[sqlx::test] + async fn pinned_cids_sources_incomplete_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v15 shape: drop the column and forget v15 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS pin_sources_incomplete") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 15") + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind("preu3oid") + .bind("preu3cid") + .bind(chrono::Utc::now().to_rfc3339()) + .execute(&pool) + .await + .unwrap(); + + state.db.run_migrations().await.expect("re-migrate"); + state + .db + .run_migrations() + .await + .expect("migrations are idempotent: a second run succeeds"); + + assert!( + !state.db.pin_sources_incomplete("preu3oid").await.unwrap(), + "a row predating the column reads COMPLETE, so the upgrade arms no fallback" + ); + state + .db + .mark_pin_sources_incomplete("preu3oid") + .await + .expect("mark after upgrade"); + assert!( + state.db.pin_sources_incomplete("preu3oid").await.unwrap(), + "the v15 column is present and writable after the upgrade" + ); + } + /// #173 (jatmn round 8, F2 — load-bearing): a legacy `pinned_cids` row keyed on a /// PROVIDER CID (Pinata/Kubo dag-pb — every release before this branch stored the /// provider CID as the resolver key, not the raw-content CID) must NOT serve raw git From 412d04e12dcad400d770cb62f4abe807b625910e Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:53:26 -0500 Subject: [PATCH 19/77] fix(node): sweep legacy provider-CID pins instead of waiting for a re-push `repair_legacy_provider_cid` had exactly one trigger: the already-pinned skip branch inside `pin_new_objects`, which a push only reaches when it re-carries the object. Normal git negotiation omits objects the node already has, so on an upgraded node that push generally never arrives. Meanwhile `list_pinned_cids` kept advertising the stored provider CID that this branch's `/ipfs/{cid}` deliberately withholds, so the node handed out CIDs it would then refuse to serve, and previously pinned data stayed unresolvable. Two halves, both needed. A detached background sweep walks `pinned_cids` in bounded batches with an inter-batch delay, resolves each row's repo from its recorded provenance, and reuses the existing repair (and its cost gate, so a row already keyed on a raw CIDv1 reads no bytes). Migration v16 adds a single-row cursor table so a restart mid-walk resumes rather than rewinding; the cursor advances on every row read, before any skip, so an unrepairable row cannot wedge the walk. A row whose bytes are gone, whose provenance is NULL, or whose repo is not on local disk is left exactly as it is. And `list_pinned_cids` now omits rows whose key is not a raw CIDv1, so the window before the sweep catches up advertises nothing the resolver withholds. The sweep resolves paths directly rather than through `repo_store.acquire`, which would pull cold repos back from Tigris and turn a repair pass into a bulk restore. Defaults are deliberately quiet: 64 rows per batch, 60s between batches. Ten tests cover the repair, the bytes-gone and unrepairable arms, the cost gate, the batch bound, cursor resumption, both states of one row through the advertise filter, and the degenerate empty and no-legacy-rows tables. Removing the filter, the batch bound, the cursor advance, the bytes-gone early return, or the inter-batch sleep each turns a test red. One pre-existing test used placeholder resolver keys that the new filter drops; its values are now real raw CIDv1s with every assertion unchanged. --- crates/gitlawb-node/src/config.rs | 62 +++ crates/gitlawb-node/src/db/mod.rs | 91 ++++ crates/gitlawb-node/src/ipfs_pin.rs | 160 ++++++ crates/gitlawb-node/src/main.rs | 37 ++ crates/gitlawb-node/src/test_support.rs | 638 +++++++++++++++++++++++- 5 files changed, 984 insertions(+), 4 deletions(-) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 5480beda..aff27d1e 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -403,6 +403,39 @@ pub struct Config { /// disables that derived bucket too. #[arg(long, env = "GITLAWB_IPFS_RATE_LIMIT", default_value_t = 600)] pub ipfs_rate_limit: usize, + + /// Rows the legacy provider-CID repair sweep reads per batch (U4, #173). + /// + /// The sweep walks every `pinned_cids` row on the node once, repairing rows that + /// releases before this branch keyed on a PROVIDER CID (Kubo dag-pb / Pinata CIDv0) + /// instead of the raw-content resolver key. This bounds one batch, so the sweep can + /// never turn into a single unbounded table scan competing with request traffic. + /// Conservative on purpose: paired with the inter-batch delay below the default is + /// ~64 rows per minute, which finishes a large pin set in hours of idle background + /// work rather than one expensive burst. Must be between 1 and 100_000. + #[arg( + long, + env = "GITLAWB_PIN_REPAIR_SWEEP_BATCH", + default_value_t = 64, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=100_000) + )] + pub pin_repair_sweep_batch: i64, + + /// Seconds the legacy provider-CID repair sweep sleeps between batches (U4, #173). + /// + /// Each batch costs an indexed range scan plus, for the legacy rows in it, a + /// `git cat-file` per row. The delay is what keeps that off the DB's and the disk's + /// critical path: the sweep is repairing rows that have been unresolvable since the + /// upgrade, so finishing slowly is fine and finishing fast at the cost of live + /// traffic is not. `0` disables the pause (test and one-off operational use only). + /// Must be between 0 and 86_400. + #[arg( + long, + env = "GITLAWB_PIN_REPAIR_SWEEP_DELAY_SECS", + default_value_t = 60, + value_parser = clap::builder::RangedU64ValueParser::::new().range(0..=86_400) + )] + pub pin_repair_sweep_delay_secs: u64, } impl Config { @@ -523,6 +556,35 @@ mod tests { ); } + /// U4 (#173): the repair sweep's bounds are conservative by default and a batch of + /// 0 (a sweep that walks nothing and never terminates) is a CLI error, not a + /// runtime hang. The delay does accept 0, for tests and one-off operational runs. + #[test] + fn pin_repair_sweep_knobs_default_conservatively() { + let c = Config::parse_from(["gitlawb-node"]); + assert_eq!(c.pin_repair_sweep_batch, 64); + assert_eq!(c.pin_repair_sweep_delay_secs, 60); + + assert!(Config::try_parse_from(["gitlawb-node", "--pin-repair-sweep-batch", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--pin-repair-sweep-batch", "100001"]).is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--pin-repair-sweep-batch", "8"]) + .pin_repair_sweep_batch, + 8 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--pin-repair-sweep-delay-secs", "0"]) + .pin_repair_sweep_delay_secs, + 0 + ); + assert!( + Config::try_parse_from(["gitlawb-node", "--pin-repair-sweep-delay-secs", "86401"]) + .is_err() + ); + } + #[test] fn ipfs_walk_per_source_defaults_and_rejects_out_of_range() { assert_eq!(Config::parse_from(["gitlawb-node"]).ipfs_walk_per_source, 4); diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index c2a6cb60..4b107da3 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -956,6 +956,26 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS pin_sources_incomplete BOOLEAN NOT NULL DEFAULT FALSE", ], }, + Migration { + version: 16, + name: "pin_repair_sweep_cursor", + stmts: &[ + // U4 (#173): the legacy provider-CID repair sweep walks `pinned_cids` in + // bounded batches over an ordered `sha256_hex` cursor. The cursor has to be + // DURABLE, or a restart rewinds the walk to the start of the table and an + // upgraded node with a large pin set never finishes repairing it. One row + // (`id = 1`, enforced by the CHECK) rather than a key-value table: there is + // exactly one sweep and no second consumer, and a real constraint beats a + // convention nobody can enforce. NEW versioned migration (never appended to + // an applied block, INV-7). No default row is inserted: an absent row is the + // "never swept" state, which the empty-string cursor start already means, so + // there is no first-run special case to get wrong. + "CREATE TABLE IF NOT EXISTS pin_repair_sweep ( + id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), + cursor TEXT NOT NULL + )", + ], + }, ]; /// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). @@ -2368,6 +2388,65 @@ impl Db { Ok(()) } + /// One ordered batch of `pinned_cids` rows strictly after `cursor`, for the U4 + /// legacy provider-CID repair sweep. Returns `(sha256_hex, cid)` ordered by + /// `sha256_hex` (the table's primary key, so the walk rides the PK index) and + /// capped at `limit` rows, which is what BOUNDS the sweep: one pass can never read + /// more than a batch, however large the pin set is. + /// + /// Deliberately NOT filtered to legacy rows in SQL. "Is this a raw CIDv1" is a + /// multibase+codec decode (`is_raw_cidv1`), which Postgres cannot express, and a + /// prefix-match approximation would silently mis-classify keys under a different + /// multihash. The caller applies the real predicate, so `limit` bounds rows READ + /// (the DB cost), not rows repaired. + pub async fn pinned_cids_after( + &self, + cursor: &str, + limit: i64, + ) -> Result> { + let rows = sqlx::query( + "SELECT sha256_hex, cid FROM pinned_cids + WHERE sha256_hex > $1 + ORDER BY sha256_hex + LIMIT $2", + ) + .bind(cursor) + .bind(limit) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| (r.get::("sha256_hex"), r.get::("cid"))) + .collect()) + } + + /// Where the U4 repair sweep's walk left off, or `""` before it has ever run. + /// Empty string sorts below every hex oid, so a first run and a rewound run are + /// the same code path (`sha256_hex > ''` is the whole table). + pub async fn pin_repair_cursor(&self) -> Result { + let row = sqlx::query("SELECT cursor FROM pin_repair_sweep WHERE id = 1") + .fetch_optional(&self.pool) + .await?; + Ok(row + .map(|r| r.get::("cursor")) + .unwrap_or_default()) + } + + /// Persist the sweep's walk position. Written after every batch, so a restart + /// resumes rather than re-walking the table from the beginning. A rewrite is a + /// plain upsert: the sweep is the single writer, and re-repairing an + /// already-repaired row is a no-op anyway (the codec cost gate spares it). + pub async fn set_pin_repair_cursor(&self, cursor: &str) -> Result<()> { + sqlx::query( + "INSERT INTO pin_repair_sweep (id, cursor) VALUES (1, $1) + ON CONFLICT (id) DO UPDATE SET cursor = EXCLUDED.cursor", + ) + .bind(cursor) + .execute(&self.pool) + .await?; + Ok(()) + } + /// The repository a pinned object was recorded from (`pinned_cids.repo_id`), /// or `None` for a legacy pin (recorded before provenance existed) or an /// unpinned oid. `GET /ipfs/{cid}` uses this to gate+serve the ONE source @@ -2642,6 +2721,17 @@ impl Db { Ok(row.map(|r| r.get("recipients_tag"))) } + /// Every pinned object this node ADVERTISES (`GET /api/v1/ipfs/pins`). + /// + /// U4 (#173): rows still keyed on a legacy PROVIDER CID (Kubo dag-pb / Pinata + /// CIDv0, written by releases before this branch) are withheld from the listing. + /// The `/ipfs/{cid}` resolver recomputes the raw-content CID from the object bytes + /// and refuses any row whose stored key does not match, so advertising the legacy + /// key hands a client a CID this node deliberately will not serve. The background + /// repair sweep rewrites those rows to the raw key, and each one reappears here the + /// moment it is repaired. Filtering is done in Rust because the raw-CIDv1 test is a + /// multibase+codec decode (`is_raw_cidv1`), not something SQL can express; it is the + /// SAME predicate the repair path uses as its cost gate, so the two cannot drift. pub async fn list_pinned_cids(&self) -> Result> { let rows = sqlx::query( "SELECT sha256_hex, cid, pinned_at, pinata_cid FROM pinned_cids ORDER BY pinned_at DESC", @@ -2650,6 +2740,7 @@ impl Db { .await?; Ok(rows .into_iter() + .filter(|r| gitlawb_core::cid::is_raw_cidv1(r.get::<&str, _>("cid"))) .map(|r| PinnedCidRecord { sha256_hex: r.get("sha256_hex"), cid: r.get("cid"), diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 0e58e27e..1f0cc65a 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -99,6 +99,166 @@ async fn repair_legacy_provider_cid( db.repair_legacy_provider_cid(sha, &raw, &stored).await } +/// What one sweep pass (or a whole sweep run) did. `scanned` counts `pinned_cids` +/// rows READ, which is the quantity the batch size bounds; `repaired` counts rows +/// whose key was actually rewritten to the raw CID. +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct SweepStats { + pub scanned: usize, + pub repaired: usize, + pub passes: usize, +} + +/// One bounded pass of the U4 sweep: read at most `batch` `pinned_cids` rows after +/// the persisted cursor, repair the legacy ones, and persist the new cursor. +/// +/// The batch is what bounds the pass. It caps rows READ, not rows repaired, because +/// the legacy predicate is a codec decode SQL cannot express; a table of raw rows +/// therefore costs one indexed range scan per pass and nothing else. +/// +/// The cursor advances to the LAST row read whatever happened to each row, including +/// rows that were skipped as unrepairable. A cursor that only advanced on success +/// would re-read the same unrepairable row on every pass and the sweep would never +/// reach the rows behind it. +async fn sweep_pass( + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + batch: i64, + db: &crate::db::Db, +) -> Result { + let cursor = db.pin_repair_cursor().await?; + let rows = db.pinned_cids_after(&cursor, batch).await?; + let scanned = rows.len(); + let mut repaired = 0usize; + let mut last = cursor; + + for (sha, stored) in rows { + // Advance FIRST: every path below this line may skip the row, and none of them + // may wedge the walk (scenario 7). + last = sha.clone(); + // Same cost gate as the skip-path repair: a canonical raw CIDv1 key is already + // the resolver key, so it reads no bytes and resolves no repo. + if gitlawb_core::cid::is_raw_cidv1(&stored) { + continue; + } + // Resolve the row's repo from its recorded provenance (first-pinner plus the + // bounded additional source set). An empty set is a pin recorded before + // provenance existed: nothing to read the bytes from, so skip it. + let sources = match db.pin_sources_for_oid(&sha).await { + Ok(s) => s, + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "sweep: failed to read pin sources"); + continue; + } + }; + for repo_id in sources { + let repo = match db.get_repo_by_id(&repo_id).await { + Ok(Some(r)) => r, + // The repo row is gone: a later source may still hold the bytes. + Ok(None) => continue, + Err(e) => { + tracing::warn!(repo_id = %repo_id, err = %e, "sweep: failed to read repo"); + continue; + } + }; + // Derive the LOCAL disk path rather than going through `repo_store.acquire`. + // The sweep is opportunistic background maintenance over every pinned row on + // the node, so it must never pull a cold repo back from remote storage: that + // would turn a repair pass into a bulk restore. A repo that is not on local + // disk simply reads no bytes here and stays withheld for a later pass or a + // re-push, which is the same non-destructive outcome as missing bytes. + let repo_path = + crate::git::store::repo_disk_path(repos_dir, &repo.owner_did, &repo.name); + if let Err(e) = + repair_legacy_provider_cid(&repo_path, git_bin, git_timeout, &sha, db).await + { + tracing::warn!(sha = %sha, err = %e, "sweep: legacy provider-CID repair failed"); + continue; + } + // `repair_legacy_provider_cid` is best-effort and silent about which of its + // outcomes it took (bytes gone, read failed, rewritten), so read the key back + // to decide whether to stop trying sources. A no-op re-read on an + // already-repaired row is one indexed lookup. + match db.cid_for_oid(&sha).await { + Ok(Some(c)) if gitlawb_core::cid::is_raw_cidv1(&c) => { + repaired += 1; + break; + } + Ok(_) => continue, + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "sweep: failed to re-read repaired key"); + continue; + } + } + } + } + + db.set_pin_repair_cursor(&last).await?; + Ok(SweepStats { + scanned, + repaired, + passes: 1, + }) +} + +/// Test seam for a single bounded pass (scenarios 4 and 5 drive passes by hand to +/// observe the batch bound and the restart-resumes-from-cursor behavior). +#[cfg_attr(not(test), allow(dead_code))] +pub(crate) async fn sweep_legacy_provider_cids_once( + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + batch: i64, + db: &crate::db::Db, +) -> Result { + sweep_pass(repos_dir, git_bin, git_timeout, batch, db).await +} + +/// U4 (#173): the one-shot legacy provider-CID migration sweep. +/// +/// Releases before this branch stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) +/// in `pinned_cids.cid`. This branch's `/ipfs/{cid}` resolver recomputes the raw +/// content CID and withholds any row whose key does not match, so those rows are +/// unresolvable. The opportunistic repair on the already-pinned skip path only fires +/// when a later push re-carries the object, and normal git negotiation omits objects +/// the node already has, so on an upgraded node that push generally never comes. This +/// walks the table instead. +/// +/// Runs until a pass comes back short of a full batch, which is the end of the table. +/// Sleeps `delay` between full batches so it cannot monopolize the DB, and persists +/// its cursor every pass so a restart continues instead of rewinding. Errors reading +/// or repairing an individual row are warn-and-skip; only a failure of the batch query +/// or the cursor write ends the run, and a later run picks up from the stored cursor. +pub(crate) async fn sweep_legacy_provider_cids( + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + batch: i64, + delay: Duration, + db: &crate::db::Db, +) -> SweepStats { + let mut totals = SweepStats::default(); + loop { + let pass = match sweep_pass(repos_dir, git_bin, git_timeout, batch, db).await { + Ok(p) => p, + Err(e) => { + tracing::warn!(err = %e, "legacy provider-CID sweep pass failed; stopping"); + return totals; + } + }; + totals.scanned += pass.scanned; + totals.repaired += pass.repaired; + totals.passes += 1; + // A short batch means the ordered walk reached the end of the table. Stop here + // rather than after an extra empty pass, and do NOT sleep on the way out. + if (pass.scanned as i64) < batch { + return totals; + } + tokio::time::sleep(delay).await; + } +} + // Test-only cost-gate counter (R8, U7): how many times the opportunistic repair // read an object's bytes on the skip path. The codec gate must spare a CIDv1/raw // row this read; the counter is the both-ways guard (removing the gate reads the diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index fe18c706..648d7428 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -520,6 +520,43 @@ async fn main() -> Result<()> { }); } + // U4 (#173): one-shot legacy provider-CID repair sweep. Releases before this + // version stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`, + // and this version's `/ipfs/{cid}` resolver withholds any row whose stored key is + // not the raw-content CID. The opportunistic repair on the pin path only fires when + // a push re-carries the object, which normal git negotiation makes it not do, so + // those rows need a walk. DETACHED, never on the boot path: the server below starts + // and serves while this runs, and the sweep's own batch bound plus inter-batch delay + // keep it off the DB's critical path. Its cursor is durable, so a restart mid-walk + // resumes instead of rewinding. + { + let db = state.db.clone(); + let repos_dir = config.repos_dir.clone(); + let git_bin = state.git_bin.clone(); + let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs); + let batch = config.pin_repair_sweep_batch; + let delay = std::time::Duration::from_secs(config.pin_repair_sweep_delay_secs); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + tokio::select! { + stats = ipfs_pin::sweep_legacy_provider_cids( + &repos_dir, &git_bin, git_timeout, batch, delay, &db, + ) => { + if stats.repaired > 0 { + tracing::info!( + scanned = stats.scanned, + repaired = stats.repaired, + "legacy provider-CID sweep finished" + ); + } + } + // Shutdown mid-walk simply drops the run; the persisted cursor means the + // next boot picks up where this one stopped. + _ = shutdown_rx.changed() => {} + } + }); + } + let router = server::build_router(state.clone()); // Re-register the socket bound at startup — same fd, so there was never a // moment with the port closed between the degraded and full servers. diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index fd90e46a..47428248 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -3523,11 +3523,17 @@ mod tests { async fn record_pinata_cid_stores_and_coalesces_provenance(pool: PgPool) { let state = test_state(pool).await; + // Real raw-CIDv1 resolver keys, as the pin paths write them: `list_pinned_cids` + // withholds any row keyed on a non-raw (legacy provider) value (U4, #173), so a + // placeholder string here would be filtered out and make the assertions vacuous. + let raw1 = gitlawb_core::cid::Cid::from_git_object_bytes(b"pinata raw 1").to_string(); + let local2 = gitlawb_core::cid::Cid::from_git_object_bytes(b"local raw 2").to_string(); + // A new row created via the pinata path carries provenance, and stores the // raw CID in `cid` with the provider CID in `pinata_cid`. state .db - .record_pinata_cid("po1", "rawcid1", "pcid1", Some("repoA")) + .record_pinata_cid("po1", &raw1, "pcid1", Some("repoA")) .await .unwrap(); assert_eq!( @@ -3542,7 +3548,7 @@ mod tests { .into_iter() .find(|r| r.sha256_hex == "po1") .expect("po1 row exists"); - assert_eq!(po1.cid, "rawcid1", "resolver-key cid is the raw CID"); + assert_eq!(po1.cid, raw1, "resolver-key cid is the raw CID"); assert_eq!( po1.pinata_cid.as_deref(), Some("pcid1"), @@ -3553,7 +3559,7 @@ mod tests { // prior local pin's `cid` is left untouched (not overwritten by the raw arg). state .db - .record_pinned_cid("po2", "localcid2", None) + .record_pinned_cid("po2", &local2, None) .await .unwrap(); state @@ -3575,7 +3581,7 @@ mod tests { .find(|r| r.sha256_hex == "po2") .expect("po2 row exists"); assert_eq!( - po2.cid, "localcid2", + po2.cid, local2, "on conflict the prior local pin's cid is left untouched" ); @@ -4273,6 +4279,630 @@ mod tests { ); } + // ---- #173 U4: legacy provider-CID migration sweep ---- + + /// Seed a legacy PROVIDER-CID `pinned_cids` row for `oid` (the pre-branch shape: + /// `cid` holds the Kubo dag-pb / Pinata key, not the raw-content resolver key). + /// Returns `(raw_cid, provider_cid)`. Raw SQL because every production helper + /// stores the already-correct raw key. + async fn seed_legacy_pin( + pool: &PgPool, + bare: &std::path::Path, + oid: &str, + repo_id: Option<&str>, + ) -> (String, String) { + let (_ty, bytes) = crate::git::store::read_object(bare, oid) + .expect("read object bytes") + .expect("object exists in the bare repo"); + let raw = gitlawb_core::cid::Cid::from_git_object_bytes(&bytes).to_string(); + let provider = legacy_dagpb_cid(&raw); + assert_ne!(provider, raw, "the legacy key differs from the raw key"); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(oid) + .bind(&provider) + .bind("2020-01-01T00:00:00Z") + .bind(repo_id) + .execute(pool) + .await + .unwrap(); + (raw, provider) + } + + /// The `pinned_cids.cid` currently stored for an oid, unfiltered (unlike + /// `list_pinned_cids`, which withholds unrepaired legacy rows). + async fn stored_pin(pool: &PgPool, oid: &str) -> (String, Option) { + sqlx::query_as("SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1") + .bind(oid) + .fetch_one(pool) + .await + .unwrap() + } + + /// U4 (#173, INV-7 upgrade path): a node already at the prior-max schema (v15) gets + /// the `pin_repair_sweep` cursor table from the NEW v16 migration. Simulate the + /// pre-v16 node by dropping the table and un-applying v16, then re-migrate and + /// assert the cursor round-trips. RED before the v16 migration exists (the table is + /// never recreated, so the cursor read errors). + #[sqlx::test] + async fn pin_repair_sweep_cursor_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v16 shape: drop the table and forget v16 was applied. + sqlx::query("DROP TABLE IF EXISTS pin_repair_sweep") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 16") + .execute(&pool) + .await + .unwrap(); + + state.db.run_migrations().await.expect("migrate to v16"); + + // Absent row reads as the "never swept" start, and a write round-trips. + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + "", + "a node that has never swept starts at the beginning of the table" + ); + state.db.set_pin_repair_cursor("abc").await.unwrap(); + state.db.set_pin_repair_cursor("def").await.unwrap(); + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + "def", + "the v16 cursor table persists the walk position across writes" + ); + let rows: i64 = sqlx::query_scalar("SELECT count(*) FROM pin_repair_sweep") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(rows, 1, "the cursor is a single row, not an append log"); + } + + /// U4 scenario 1 (#173): a legacy provider-CID row with intact object bytes is + /// repaired to the raw-content resolver key by the SWEEP alone, with the old value + /// stashed in `legacy_provider_cid`. No push, no re-pin: this is the whole point of + /// U4, because normal git negotiation omits objects the node already has, so the + /// skip-branch repair's re-push trigger generally never fires on an upgraded node. + /// RED before the sweep is implemented (the row keeps its provider key). + #[sqlx::test] + async fn sweep_repairs_legacy_row_without_a_push(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["swsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("swsrc.git"); + let repo = seed_repo(&owner_did, "swsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let (raw_cid, provider_cid) = + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ) + .await; + assert_eq!(stats.repaired, 1, "the sweep repairs the one legacy row"); + + let (stored, stashed) = stored_pin(&pool, &fx.public_oid).await; + assert_eq!( + stored, raw_cid, + "the key is rewritten to the raw-content CID" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed in legacy_provider_cid" + ); + + // End to end: the repaired key is now advertised AND serves. + assert!( + state + .db + .list_pinned_cids() + .await + .unwrap() + .iter() + .any(|r| r.cid == raw_cid), + "the repaired row is advertised" + ); + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "the repaired raw key serves"); + assert!(body.contains("public bytes"), "the object's bytes serve"); + } + + /// U4 scenario 2 (#173): a legacy row whose object bytes are gone is left exactly + /// as it is by the sweep: never rewritten, never deleted. The row stays withheld + /// until the bytes come back, which is the non-destructive contract the skip-branch + /// repair already holds. + #[sqlx::test] + async fn sweep_leaves_a_bytes_gone_row_untouched(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let _fx = seed_cid_repos(&slug, &short, &["gonesrc"]); + let repo = seed_repo(&owner_did, "gonesrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // An oid whose bytes are NOT in the repo, but whose provenance resolves fine. + let phantom_oid = "d".repeat(64); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&phantom_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ) + .await; + assert_eq!(stats.repaired, 0, "an unrepairable row is not repaired"); + + let (stored, stashed) = stored_pin(&pool, &phantom_oid).await; + assert_eq!( + stored, provider_cid, + "the bytes-gone row keeps its provider CID (no destructive rewrite)" + ); + assert_eq!(stashed, None, "nothing is stashed when the bytes are gone"); + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM pinned_cids") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(count, 1, "the row is not deleted"); + } + + /// U4 scenario 3 (#173): the sweep inherits `repair_legacy_provider_cid`'s cost + /// gate, so a row already keyed on a raw CIDv1 is NEVER read for bytes. The + /// test-only `legacy_repair_reads` counter is the both-ways guard: dropping the + /// codec gate reads the raw row and trips it off zero. + #[sqlx::test] + async fn sweep_never_reads_bytes_for_a_raw_cidv1_row(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["rawsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("rawsrc.git"); + let repo = seed_repo(&owner_did, "rawsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let raw_cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + assert!( + gitlawb_core::cid::is_raw_cidv1(&raw_cid), + "the seeded row is already the canonical resolver key" + ); + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ) + .await; + assert_eq!(stats.scanned, 1, "the sweep walked the row"); + assert_eq!(stats.repaired, 0, "a raw row needs no repair"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a raw-CIDv1 row is never read for bytes (cost gate)" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the raw row is left as-is" + ); + } + + /// U4 scenario 4 (#173, BOUND): one pass reads at most `batch` rows, so it repairs + /// at most `batch` of them. The exact count is asserted, so raising or removing the + /// bound fails. This is what keeps the sweep from monopolizing the DB on a node + /// with a large `pinned_cids` table. + #[sqlx::test] + async fn sweep_one_pass_is_bounded_by_the_batch_size(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["batchsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("batchsrc.git"); + let repo = seed_repo(&owner_did, "batchsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // Five legacy rows, batch of two. + for oid in [ + &fx.public_oid, + &fx.secret_oid, + &fx.public_tree_oid, + &fx.secret_tree_oid, + &fx.commit_oid, + ] { + seed_legacy_pin(&pool, &bare, oid, Some(&repo.id)).await; + } + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 2, + &state.db, + ) + .await + .expect("one pass runs"); + assert_eq!(stats.scanned, 2, "one pass reads exactly the batch size"); + assert_eq!(stats.repaired, 2, "one pass repairs at most the batch size"); + + let repaired: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pinned_cids WHERE legacy_provider_cid IS NOT NULL", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(repaired, 2, "exactly two of the five rows were rewritten"); + } + + /// U4 scenario 5 (#173, RESUMPTION): the walk cursor persists, so a sweep + /// interrupted mid-table continues from where it stopped instead of restarting. + /// Two bounded passes are driven by hand (the restart), and the second pass is + /// asserted to repair the NEXT two rows in cursor order, not the first two again. + /// The read counter proves the already-repaired rows are not re-read. + #[sqlx::test] + async fn sweep_resumes_from_the_persisted_cursor(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["resumesrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("resumesrc.git"); + let repo = seed_repo(&owner_did, "resumesrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let mut oids = vec![ + fx.public_oid.clone(), + fx.secret_oid.clone(), + fx.public_tree_oid.clone(), + fx.secret_tree_oid.clone(), + ]; + for oid in &oids { + seed_legacy_pin(&pool, &bare, oid, Some(&repo.id)).await; + } + // The cursor is an ordered walk over the `pinned_cids` primary key. + oids.sort(); + + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let pass1 = crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 2, + &state.db, + ) + .await + .expect("pass 1 runs"); + assert_eq!(pass1.repaired, 2, "pass 1 repairs the first two rows"); + + // The restart: a second pass over the SAME state must continue, not rewind. + crate::ipfs_pin::reset_legacy_repair_reads(); + let pass2 = crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 2, + &state.db, + ) + .await + .expect("pass 2 runs"); + assert_eq!(pass2.repaired, 2, "pass 2 repairs the NEXT two rows"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 2, + "pass 2 reads bytes only for the two rows it repaired; the already-repaired \ + rows are not re-read" + ); + for oid in &oids { + let (_cid, stashed) = stored_pin(&pool, oid).await; + assert!( + stashed.is_some(), + "every row is repaired after two resumed passes" + ); + } + } + + /// U4 scenario 7 (#173, cursor liveness): a row that cannot be repaired (NULL + /// provenance, or a provenance whose repo row is gone) is skipped AND the cursor + /// still advances past it. With `batch = 1` the two unrepairable rows sort first, + /// so a cursor that failed to advance would re-read the same row forever and never + /// reach the repairable row behind them. The outer timeout turns that into a + /// FAILURE rather than a hung suite. + #[sqlx::test] + async fn sweep_advances_past_unrepairable_rows(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["skipsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("skipsrc.git"); + let repo = seed_repo(&owner_did, "skipsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // Two blockers that sort ahead of any real 64-hex oid: one with NULL + // provenance, one naming a repo row that no longer exists. + let null_prov_oid = "0".repeat(64); + let ghost_repo_oid = format!("{}1", "0".repeat(63)); + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + for (oid, prov) in [ + (&null_prov_oid, None), + (&ghost_repo_oid, Some("repo-that-is-gone")), + ] { + let raw = gitlawb_core::cid::Cid::from_git_object_bytes(oid.as_bytes()).to_string(); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(oid) + .bind(legacy_dagpb_cid(&raw)) + .bind("2020-01-01T00:00:00Z") + .bind(prov) + .execute(&pool) + .await + .unwrap(); + } + assert!( + null_prov_oid < fx.public_oid && ghost_repo_oid < fx.public_oid, + "the blockers really do sort ahead of the repairable row" + ); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 1, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates instead of looping on an unrepairable row"); + + assert_eq!( + stats.repaired, 1, + "the sweep advanced past both blockers and repaired the row behind them" + ); + assert!( + stored_pin(&pool, &fx.public_oid).await.1.is_some(), + "the row behind the blockers is the one that got repaired" + ); + for oid in [&null_prov_oid, &ghost_repo_oid] { + assert_eq!( + stored_pin(&pool, oid).await.1, + None, + "an unrepairable row is left untouched" + ); + } + } + + /// U4 scenario 8 (#173, degenerate states): an empty `pinned_cids` table and a + /// table with zero legacy rows both complete cleanly, with no repair and no read. + #[sqlx::test] + async fn sweep_completes_on_degenerate_tables(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + // Empty table. + crate::ipfs_pin::reset_legacy_repair_reads(); + let empty = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 4, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates on an empty table"); + assert_eq!( + (empty.scanned, empty.repaired), + (0, 0), + "an empty table is a clean no-op" + ); + + // Zero legacy rows: every row already carries the canonical raw key. + let fx = seed_cid_repos(&slug, &short, &["degensrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("degensrc.git"); + let repo = seed_repo(&owner_did, "degensrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + for oid in [&fx.public_oid, &fx.secret_oid, &fx.commit_oid] { + pin_cid_for_repo(&bare, oid, &state.db, &repo.id).await; + } + + let clean = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 4, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates on a table with no legacy rows"); + assert_eq!(clean.scanned, 3, "every row is walked"); + assert_eq!(clean.repaired, 0, "nothing needs repair"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "no object bytes are read when no row is legacy" + ); + } + + /// U4 (#173, BOUND): the inter-batch delay is real, observed by wall clock. Five + /// rows at a batch of two means two full batches and a trailing partial one, so the + /// run sleeps twice. Without the sleep the whole run is sub-millisecond DB work and + /// a node's `pinned_cids` table gets walked as fast as Postgres will answer. + #[sqlx::test] + async fn sweep_sleeps_between_batches(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["delaysrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("delaysrc.git"); + let repo = seed_repo(&owner_did, "delaysrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + for oid in [ + &fx.public_oid, + &fx.secret_oid, + &fx.public_tree_oid, + &fx.secret_tree_oid, + &fx.commit_oid, + ] { + pin_cid_for_repo(&bare, oid, &state.db, &repo.id).await; + } + + let delay = std::time::Duration::from_millis(150); + let started = std::time::Instant::now(); + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 2, + delay, + &state.db, + ) + .await; + let elapsed = started.elapsed(); + + assert_eq!( + stats.passes, 3, + "five rows at a batch of two is three passes" + ); + assert!( + elapsed >= delay * 2, + "the run sleeps once between each pair of full batches: {elapsed:?} < {:?}", + delay * 2 + ); + } + + /// U4 scenario 6 (#173): `list_pinned_cids` never advertises a key the `/ipfs` + /// resolver would withhold. The resolver recomputes the raw CIDv1 from the object + /// bytes and 404s any row keyed on a legacy PROVIDER CID, so advertising that key + /// hands clients a CID this node deliberately refuses. Both states of ONE row are + /// asserted (omitted while legacy, present once repaired) so the test cannot pass + /// by accident. RED before the `is_raw_cidv1` filter lands: the legacy row is + /// advertised. + #[sqlx::test] + async fn list_pinned_cids_omits_unrepaired_legacy_row(pool: PgPool) { + let state = test_state(pool).await; + + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"u4 advertise bytes").to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + let oid = "c".repeat(64); + state + .db + .record_pinned_cid(&oid, &provider_cid, None) + .await + .unwrap(); + + let listed = state.db.list_pinned_cids().await.unwrap(); + assert!( + !listed.iter().any(|r| r.sha256_hex == oid), + "an unrepaired legacy provider-CID row is not advertised" + ); + + // Same row, repaired: it comes back, keyed on the raw CID the resolver serves. + state + .db + .repair_legacy_provider_cid(&oid, &raw_cid, &provider_cid) + .await + .unwrap(); + let listed = state.db.list_pinned_cids().await.unwrap(); + let rec = listed + .iter() + .find(|r| r.sha256_hex == oid) + .expect("the repaired row is advertised again"); + assert_eq!( + rec.cid, raw_cid, + "the advertised key is the raw-content resolver key" + ); + } + /// #173 (provenance-path throttle): a walk-requiring provenanced candidate whose /// per-IP walk quota is spent returns 429 (the provenance arm's Throttled outcome, /// then the fall-through). quota=1, keyed on XFF. The first reader request runs the From 752fd19b6d17dcf2a61e43a1c4203d5fa40497f8 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:47:56 -0500 Subject: [PATCH 20/77] fix(node): stop the lock pool from being occupied by spinners and racing the reaper Three regressions from the previous four commits, each demonstrated before being fixed. The advisory-lock retry loop checked its connection out before spinning, so a caller that lost the lock held a pool connection through all sixty one-second attempts. The old pool-per-query shape returned it between iterations, so a spinner occupied nothing. That matters more than the pool sizing suggested: `acquire_write` has three non-push callers (two in issues, one in pulls) that hold no concurrency permit, and the issue routes carry no rate limit, so any self-minted DID could fire enough concurrent closes at one repo to strand the pool and fail authenticated pushes on every repo. The checkout now happens per attempt and a losing attempt returns the connection before sleeping, so only the winner retains one. Lock-pool exhaustion also carries its own error now and sheds 503 with Retry-After instead of surfacing as a 500. Holding the lock on one connection then meant a client disconnect released it early: the handler future dies mid-receive-pack, the guard drops without `release`, and `after_release` frees the lock while the detached reaper is still giving the process group its SIGTERM grace. A second writer was admitted 90ms after the drop, well inside that window, which is the invariant smart_http already documents. The write guard now rides the AdmissionGuard into the reaper, the same seam the admission permits already use, so the lock outlives the group. The success path still reclaims the guard and releases it synchronously, and a dropped guard performs no Tigris upload, so an interrupted push cannot publish a half-applied repo. The requeue loop's exit on an exhausted re-read discarded a push that coalesced during the retry window: `requeue_or_release` had cleared the dirty bit without marking the guard released, so `Drop` removed the key and no pass was ever attempted. It now falls through with nothing to replicate and lets the tail's atomic check-and-clear decide. A lap only happens when a push actually coalesced and each pays a full bounded re-read, so a sustained outage still terminates, which is tested with ten thousand injected faults under a watchdog rather than argued. Also clamps the lock pool's derived size, which previously followed max_concurrent_git_pushes up to its million-connection ceiling, and documents the node's total connection budget on that knob. --- crates/gitlawb-node/src/api/repos.rs | 462 +++++++++++++++++++++- crates/gitlawb-node/src/config.rs | 11 + crates/gitlawb-node/src/git/repo_store.rs | 257 ++++++++++-- crates/gitlawb-node/src/git/smart_http.rs | 32 +- crates/gitlawb-node/src/main.rs | 65 ++- crates/gitlawb-node/src/test_support.rs | 192 +++++++++ 6 files changed, 969 insertions(+), 50 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 35c49dd8..c8e9f2a6 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1146,10 +1146,25 @@ async fn run_post_push_replication( is_public, owner_did, } => (Some(rules), is_public, owner_did), - // The repo is gone (terminal) or the re-read never succeeded (already logged - // at ERROR). Either way there is no fresh state to act on, so exit; the guard - // Drop removes the key so the repo is never locked out of a future task. - RequeueRefresh::Gone | RequeueRefresh::Failed => break, + // The repo is gone: terminal, and there will never be fresh state to act on. + // Exit; the guard Drop removes the key so the repo is never locked out of a + // future task. + RequeueRefresh::Gone => break, + // The re-read never succeeded (already logged at ERROR). Do NOT exit here + // (#173 F3): `requeue_or_release` cleared the dirty bit without marking the + // guard released, so a push that coalesced during the ~350ms retry window is + // recorded only in a flag that `EncryptInflightGuard::drop` then removes, + // with no pass ever attempted and no reconciliation sweep to re-derive it. + // Fall through with nothing to replicate and let the TAIL's atomic + // check-and-clear make the call: dirty -> one more lap (which re-reads + // afresh), clean -> exit. Bounded under a sustained outage, because a lap + // only happens when a push actually coalesced and each one pays a full + // bounded re-read (REQUEUE_REREAD_MAX_ATTEMPTS with backoff), so this cannot + // hot-spin. + RequeueRefresh::Failed => { + withheld = None; + continue; + } }; let (_announce, r_withheld) = replication_withheld_set( r_rules.clone(), @@ -1225,6 +1240,23 @@ pub(crate) async fn run_post_push_replication_for_test( .await; } +/// Map an `acquire_write` failure to the right `AppError`. An exhausted repo write-lock +/// POOL is a capacity signal, not a broken repo, so it sheds 503 + Retry-After the same +/// way the admission caps around it do; it used to fall into the generic git 500, which +/// tells the client nothing about retrying (#173 F1). Anything else stays a git error. +fn acquire_write_app_error(err: &anyhow::Error, repo: &str) -> AppError { + if err + .downcast_ref::() + .is_some() + { + tracing::warn!(repo = %repo, err = %err, "write-lock pool exhausted; shedding with 503"); + AppError::Overloaded("git write locks at capacity, retry shortly".into()) + } else { + tracing::error!(repo = %repo, err = %err, "acquire_write failed"); + AppError::Git(err.to_string()) + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -1678,10 +1710,7 @@ pub async fn git_receive_pack( tracing::warn!(repo = %name, "acquire_write timed out; shedding with 503"); AppError::Overloaded("git service acquisition timed out, retry shortly".into()) })? - .map_err(|e| { - tracing::error!(repo = %name, err = %e, "acquire_write failed"); - AppError::Git(e.to_string()) - })?; + .map_err(|e| acquire_write_app_error(&e, name))?; let disk_path = guard.path().to_path_buf(); tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); @@ -1691,7 +1720,24 @@ pub async fn git_receive_pack( // instant a disconnect drops this future while the detached reaper runs (#174 P1-a). // The handler keeps no copy. This is independent of the write-lock `guard.release` // below: admission tracks the git process lifetime, the write lock tracks the repo. - let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + // + // The WRITE LOCK rides the same seam (#173 F2). `guard.release(..)` below is only + // reached if `receive_pack` returns, so on a client disconnect the guard would drop + // with the future and the lock pool's `after_release` hook would free the advisory + // lock immediately, while `KillGroupOnDrop`'s detached reaper is still giving the + // group its ~2s SIGTERM grace. A second push admitted in that window puts two + // `git receive-pack` groups on one repo, which is exactly what the timeout path + // reaps to prevent ("a caller releasing a write lock can't race them"). Sharing the + // guard rather than moving it outright is what lets the SUCCESS path still reclaim + // it for the Tigris upload: the copy retained here can only DELAY release, never + // perform it early, because the handler reaches the take below only after + // `receive_pack` has returned (group reaped or disarmed). On the disconnect path + // this copy dies with the future and the reaper's copy is last, so the lock frees + // after the reap with no upload, which is the release(success = false) semantics an + // interrupted push must have. + let guard = std::sync::Arc::new(std::sync::Mutex::new(Some(guard))); + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit) + .with_hold(std::sync::Arc::clone(&guard)); let receive_result = smart_http::receive_pack( &state.git_bin, &disk_path, @@ -1704,7 +1750,12 @@ pub async fn git_receive_pack( // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push // succeeded; uploading a half-applied repo would propagate corruption. - guard.release(receive_result.is_ok()).await; + let reclaimed = guard + .lock() + .expect("repo write-lock mutex poisoned") + .take() + .expect("the write lock is only taken here, and only once"); + reclaimed.release(receive_result.is_ok()).await; let result = receive_result.map_err(|e| { let app = git_service_app_error(&e); @@ -4684,6 +4735,397 @@ mod tests { ); } + /// Reproduce `repo_store::advisory_lock_key` (private there) so a test can probe the + /// exact key `acquire_write` derives. + #[cfg(unix)] + fn write_lock_key(owner_slug: &str, repo_name: &str) -> i64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + owner_slug.hash(&mut hasher); + repo_name.hash(&mut hasher); + hasher.finish() as i64 + } + + #[cfg(unix)] + fn pid_alive(pid: i32) -> bool { + // SAFETY: kill(2) with signal 0 only probes; it takes integers and borrows no + // Rust memory. + unsafe { libc::kill(pid, 0) == 0 } + } + + /// SIGKILL the recorded pids if the test unwinds, so a RED run leaks no orphan. + #[cfg(unix)] + struct KillOnPanic(Vec); + #[cfg(unix)] + impl Drop for KillOnPanic { + fn drop(&mut self) { + for pid in &self.0 { + // SAFETY: as above. + unsafe { + libc::kill(*pid, libc::SIGKILL); + } + } + } + } + + /// Is the repo write lock takeable from an INDEPENDENT session right now? Session + /// advisory locks are re-entrant within their own session, so this must not run on + /// any connection the code under test might be using. + #[cfg(unix)] + async fn write_lock_is_takeable(pool: &sqlx::PgPool, key: i64) -> bool { + let mut probe = pool.acquire().await.expect("probe connection"); + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *probe) + .await + .expect("probe try-lock"); + if taken.0 { + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *probe) + .await + .expect("probe unlock"); + } + taken.0 + } + + /// #173 F2 (RED-before/GREEN-after): on a CLIENT DISCONNECT the repo write lock must + /// stay held until the receive-pack process group is confirmed reaped. + /// + /// The handler's `guard.release(..)` line is only reached if `receive_pack` returns. + /// When the request future is dropped mid-push the guard drops instead, and (since + /// #173 U1 gave the lock pool an `after_release` hook) that FREES the advisory lock + /// immediately, while `KillGroupOnDrop`'s detached reaper is still giving the group + /// its ~2s SIGTERM grace. A second `acquire_write` admitted inside that window puts + /// two `git receive-pack` groups on one repo. `smart_http.rs` states the invariant + /// the other way round on the timeout path: "a caller releasing a write lock can't + /// race them". + /// + /// Real seam, not a stand-in: the production `git_receive_pack` handler, a fake git + /// whose descendant IGNORES SIGTERM (so the group genuinely survives the grace and + /// the window is ~2s wide, not a scheduling artifact), and the lock probed from an + /// independent session. RED before the fix: the lock is takeable while the group is + /// still alive. GREEN after: takeable only once the group is gone. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_disconnect_holds_the_write_lock_until_the_group_is_reaped( + pool: sqlx::PgPool, + ) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let owner = "z6disc"; + let name = "dc1"; + let repos_dir = tempfile::TempDir::new().unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // The leader dies on the group SIGTERM; its descendant traps SIGTERM and loops + // (bounded at ~30s so a RED run leaks nothing permanent), so the group is only + // gone once the reaper escalates to SIGKILL. The descendant inherits the stdout + // pipe, which keeps drive_git_child's read_to_end pending until we drop. + let body = format!( + "#!/bin/sh\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n", + descfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_bin = git_bin; + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + None, + crate::git::repo_store::build_lock_pool(&pool, 4, std::time::Duration::from_secs(5)), + ); + let mut cfg = (*state.config).clone(); + // Long enough that the git-service timeout is never what ends this push; the + // disconnect is. + cfg.git_service_timeout_secs = 600; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6disc-dc1", None, false) + .await + .unwrap(); + + // The mirror row stores the short owner as owner_did, so the slug is the owner. + let key = write_lock_key(&owner.replace([':', '/'], "_"), name); + // Probe from a pool that is NOT the store's lock pool and NOT the harness pool. + let probe = sqlx::postgres::PgPoolOptions::new() + .max_connections(2) + .connect_lazy_with((*pool.connect_options()).clone()); + + assert!( + write_lock_is_takeable(&probe, key).await, + "the write lock must be free before the push" + ); + + // Drive the handler a slice at a time until the fake git's SIGTERM-ignoring + // descendant records its pid, i.e. receive-pack is genuinely running under the + // write lock. `Ok(_)` means the handler returned early; stop polling then, since + // re-polling a completed future panics. + // + // Retried on a miss for the same reason `smart_http`'s disconnect tests retry: + // under `cargo test` fork-storm load a freshly written fake `git` can transiently + // fail to exec (ETXTBSY, a concurrent worker forked while its write fd was open), + // which leaves no pid. A losing attempt's future is dropped, which reaps whatever + // spawned and releases its write lock, so retries do not leak. The winning + // attempt's future is kept PENDING: dropping it below is the disconnect under test. + const SPAWN_ATTEMPTS: u64 = 12; + let (fut, desc) = { + let mut attempt = 0u64; + loop { + attempt += 1; + let _ = std::fs::remove_file(&descfile); + let mut fut = Box::pin(git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkDisconnectWriteLockProofDidAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some( + "203.0.113.81:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + let mut found: Option = None; + for _ in 0..500 { + let finished = + tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut) + .await + .is_ok(); + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + found = Some(p); + break; + } + if finished { + break; + } + } + match found { + Some(p) => break (fut, p), + None => { + drop(fut); + assert!( + attempt < SPAWN_ATTEMPTS, + "the push never reached receive-pack after {SPAWN_ATTEMPTS} \ + attempts (persistent failure, not a transient runner miss)" + ); + tokio::time::sleep(std::time::Duration::from_millis(100 * attempt)).await; + } + } + } + }; + let _cleanup = KillOnPanic(vec![desc]); + assert!( + pid_alive(desc), + "the receive-pack group must be running before the disconnect" + ); + assert!( + !write_lock_is_takeable(&probe, key).await, + "the write lock must be held while receive-pack runs" + ); + + // Client disconnect: drop the request future mid-receive-pack. + drop(fut); + + let mut takeable_while_group_alive = false; + let mut freed_after_reap = false; + for _ in 0..800 { + let takeable = write_lock_is_takeable(&probe, key).await; + let group_alive = pid_alive(desc); + if takeable && group_alive { + takeable_while_group_alive = true; + } + if takeable && !group_alive { + freed_after_reap = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + // Clean up regardless so a RED run leaves no orphan behind. + // SAFETY: kill(2) takes integers only. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + !takeable_while_group_alive, + "the repo write lock was takeable while a receive-pack group was still alive \ + on that repo: a second push can enter and two git receive-pack groups run \ + against one repo (#173 F2)" + ); + assert!( + freed_after_reap, + "the write lock must be released once the disconnected push's group is reaped" + ); + } + + /// #173 F2, the other half: carrying the write lock through the admission seam must + /// NOT cost the success path its `release(true)`. A push that completes normally has + /// to reclaim the lock and release it explicitly (that is what performs the Tigris + /// upload), synchronously, not leave it to the pool's `after_release` net. The lock + /// is probed immediately after the handler returns, with no polling, so a fix that + /// only ever dropped the guard would fail here. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_success_reclaims_and_releases_the_write_lock(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let owner = "z6succ"; + let name = "sc1"; + let repos_dir = tempfile::TempDir::new().unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + // A receive-pack that succeeds. It DRAINS stdin first: exiting while the handler + // is still writing the request body would EPIPE that write, which + // `drive_git_child` surfaces as an error after a successful exit status, making + // the push fail for a reason that has nothing to do with the lock under test. + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\ncat >/dev/null\nexit 0\n"); + + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_bin = git_bin; + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + None, + crate::git::repo_store::build_lock_pool(&pool, 4, std::time::Duration::from_secs(5)), + ); + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6succ-sc1", None, false) + .await + .unwrap(); + + let key = write_lock_key(&owner.replace([':', '/'], "_"), name); + let probe = sqlx::postgres::PgPoolOptions::new() + .max_connections(2) + .connect_lazy_with((*pool.connect_options()).clone()); + + // Retried ONLY on the ETXTBSY exec race a freshly written fake `git` hits under + // fork-storm load (a concurrent test worker forked while its write fd was open). + // Narrow on purpose: any other failure still fails the assertion below loudly. + const SPAWN_ATTEMPTS: u64 = 12; + let mut result = None; + for attempt in 1..=SPAWN_ATTEMPTS { + let outcome = tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkPushSuccessReleaseProofDidAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some( + "203.0.113.83:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("the push must return"); + let exec_race = + matches!(&outcome, Err(AppError::Git(m)) if m.contains("Text file busy")); + if exec_race && attempt < SPAWN_ATTEMPTS { + tokio::time::sleep(std::time::Duration::from_millis(100 * attempt)).await; + continue; + } + result = Some(outcome); + break; + } + let result = result.expect("one attempt must have produced an outcome"); + assert!( + result.is_ok(), + "the fake receive-pack succeeds, so the handler must too; got {result:?}" + ); + + // No polling: `release` unlocks on the connection that took the lock, so the + // lock is free the instant the handler returns. Falling back to the async + // `after_release` net would not satisfy this. + assert!( + write_lock_is_takeable(&probe, key).await, + "a completed push must reclaim its write lock and release it synchronously" + ); + } + + /// #173 F1 (RED-before/GREEN-after): an exhausted repo write-lock POOL is a capacity + /// signal, so the push must shed 503 + Retry-After (Overloaded) like every other + /// admission path here, not report a 500 git error. Both directions: the shed with + /// the single lock-pool connection occupied by a guard on a DIFFERENT repo (so this + /// is pool capacity, not advisory-lock contention), and the must-not case once that + /// connection is back. Before the fix `acquire_write`'s checkout failure fell into + /// the generic `AppError::Git` arm (500, no Retry-After). + #[sqlx::test] + async fn receive_pack_lock_pool_exhaustion_sheds_503_not_500(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let owner = "z6lockpool"; + let name = "lp1"; + let mut state = crate::test_support::test_state(pool.clone()).await; + // One lock-pool connection, short checkout timeout so the exhaustion surfaces + // promptly rather than at the handler's own acquire deadline. + state.repo_store = crate::git::repo_store::RepoStore::new( + std::path::PathBuf::from("/tmp/gitlawb-lockpool-shed"), + None, + crate::git::repo_store::build_lock_pool(&pool, 1, std::time::Duration::from_secs(1)), + ); + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6lockpool-lp1", None, false) + .await + .unwrap(); + + let did = "did:key:z6MkLockPoolShedProofDidAAAAAAAAAAAAAAAAAA"; + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + + // Occupy the only lock-pool connection with a write on an UNRELATED repo. + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + matches!(shed, Err(AppError::Overloaded(_))), + "an exhausted lock pool must shed 503 + Retry-After, not a 500 git error; \ + got {shed:?}" + ); + + // MUST-NOT: with the pool free again, the push is not shed as capacity (it fails + // later on the nonexistent on-disk repo, which is a git error, not Overloaded). + held.release(false).await; + let admitted = git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.72:5000".parse().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, a push must not be shed as capacity; got {admitted:?}" + ); + } + /// #174 U5 (P1-e, RED-before/GREEN-after): the post-push encryption walk acquires a /// `git_encrypt_semaphore` permit before running, so completed pushes cannot spawn /// unbounded concurrent full-history walks. With the pool exhausted the gated walk diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index aff27d1e..0729e04a 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -307,6 +307,17 @@ pub struct Config { /// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value /// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI /// error rather than a boot-time panic). + /// + /// CONNECTION BUDGET. A push holds a Postgres connection from the node's separate + /// advisory-lock pool for the whole receive-pack, and that pool is sized from this + /// knob (this value + 8, clamped to 64 in `main.rs`). The node's total ceiling is + /// therefore `db_max_connections` (default 20) + the lock pool (default 40), i.e. + /// 60 by default, and at most `db_max_connections` + 64. Size BOTH against the + /// database server's `max_connections`: `db_max_connections`' own doc predates the + /// lock pool and no longer covers most of the node's connections. The +8 headroom + /// is shared with the three non-push `acquire_write` callers (`api/issues.rs` x2, + /// `api/pulls.rs`). Raising this knob past the clamp does NOT buy more lock-pool + /// connections; pushes beyond it wait briefly and then shed a 503 + Retry-After. #[arg( long, env = "GITLAWB_MAX_CONCURRENT_GIT_PUSHES", diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 16c0378b..226b50f2 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -184,47 +184,60 @@ impl RepoStore { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); - // Check out ONE connection from the lock pool and keep it for the whole - // lock lifetime. Two reasons, both bugs we hit with `fetch_one(&pool)`: + // Acquire the Postgres advisory lock with retry, using pg_try_advisory_lock so a + // stale lock from a crashed connection can't block us indefinitely. // - // * A session-level advisory lock belongs to the CONNECTION that took - // it. Running the lock and the unlock through the pool lets them land - // on different connections, so `pg_advisory_unlock` silently returns - // false and the lock leaks, while a competing acquire that happens to - // draw the holding connection re-enters the lock and pushes to the - // same repo run concurrently. - // * Cancellation. `api/repos.rs` bounds this call with - // `tokio::time::timeout`; when it fires during the Tigris phase below - // the future is dropped after the lock was taken and before - // `RepoWriteGuard` (the only caller of `pg_advisory_unlock`) exists. - // Dropping this connection instead runs the pool's `after_release` - // hook, which clears the lock (#173). - let mut lock_conn = self - .lock_pool - .acquire() - .await - .context("checking out a lock-pool connection")?; - - // Acquire Postgres advisory lock with retry using pg_try_advisory_lock - // to avoid blocking indefinitely on stale locks from crashed connections. - let mut acquired = false; + // The connection is checked out INSIDE the loop and RETURNED before each sleep. + // Only the connection that actually took the lock is retained. Two constraints + // pull in opposite directions here, and this is what satisfies both: + // + // * Session ownership. A session-level advisory lock belongs to the CONNECTION + // that took it, so the lock and its `pg_advisory_unlock` must run on the same + // one. Running them through the pool (`fetch_one(&self.pool)`) lets them land + // on different connections: the unlock silently returns false and the lock + // leaks, while a competing acquire that happens to draw the holding + // connection re-enters the lock and two pushes to one repo run concurrently. + // Hence: keep the connection that WON. + // * Occupancy. Holding a connection across the ~60 one-second sleeps would let + // one spinning acquire park a lock-pool connection for a minute. That is not + // just a push-path concern: `api/issues.rs` and `api/pulls.rs` reach + // acquire_write holding no concurrency permit at all, so a caller could park + // the whole pool and starve authenticated pushes on every repo (#173 F1). + // Hence: return the connection when we LOSE, before sleeping. + // + // Returning a losing connection is safe with respect to the cancellation design: + // `after_release` runs `pg_advisory_unlock_all()`, a no-op on a connection that + // took nothing, so it cannot disturb a lock held by any other connection + // (proven by `returning_an_unlocked_connection_does_not_clear_another_connections_lock`). + // + // Cancellation safety is unchanged: the future can only be dropped while a + // connection is checked out, and dropping it runs the same `after_release` hook, + // which clears whatever lock it had just taken (#173 U1). + let mut lock_conn = None; for attempt in 0..60 { + let mut conn = self.lock_pool.acquire().await.map_err(|e| { + anyhow::Error::new(LockPoolBusy) + .context(format!("checking out a lock-pool connection: {e}")) + })?; let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(lock_key) - .fetch_one(&mut *lock_conn) + .fetch_one(&mut *conn) .await .context("trying advisory lock")?; if row.0 { - acquired = true; + lock_conn = Some(conn); break; } + // Lost the race: give the connection back so a spinning acquire occupies + // nothing while it waits. + drop(conn); if attempt < 59 { tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } - if !acquired { + let Some(lock_conn) = lock_conn else { anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}"); - } + }; #[cfg(test)] if let Some(stall) = self.tigris_stall { @@ -400,6 +413,19 @@ fn validate_repo_name(repo_name: &str) -> Result<()> { Ok(()) } +/// Error marker for "no lock-pool connection was available in time". +/// +/// Carried through the `anyhow` chain (like [`smart_http::GitServiceTimeout`]) so the +/// HTTP handler can `downcast_ref` it and shed a 503 + Retry-After instead of the +/// generic 500 a git error maps to: an exhausted lock pool is a CAPACITY signal, and +/// telling the client to retry shortly is the same shed semantics the surrounding +/// admission code already uses (#173 F1). +/// +/// [`smart_http::GitServiceTimeout`]: crate::git::smart_http::GitServiceTimeout +#[derive(Debug, thiserror::Error)] +#[error("no lock-pool connection available")] +pub struct LockPoolBusy; + /// Guard returned by `acquire_write()`. Holds the Postgres advisory lock and /// uploads to Tigris + releases the lock on `release()`. pub struct RepoWriteGuard { @@ -731,6 +757,183 @@ mod tests { held.release(false).await; } + /// #173 F1 (RED-before/GREEN-after). A contended `acquire_write` spins for up to + /// 60 one-second attempts. It must not OCCUPY a lock-pool connection for that whole + /// spin: `acquire_write` has non-push callers (`api/issues.rs`, `api/pulls.rs`) that + /// hold no concurrency permit, so any self-minted did:key could otherwise park a + /// connection per call and starve authenticated pushes on EVERY repo. + /// + /// Lock pool of exactly 2, two spinners. Pre-fix (checkout hoisted above the retry + /// loop) they pin both connections for the full spin and an UNCONTENDED acquire on a + /// third repo dies on the pool acquire timeout. Post-fix each spinner returns its + /// connection before sleeping, so it occupies ~0 and the uncontended acquire sails + /// through. + #[sqlx::test] + async fn a_spinning_acquire_write_does_not_occupy_a_lock_pool_connection(pool: PgPool) { + let owner = "did:key:z6MkSpinOccupancy"; + let owner_slug = owner.replace([':', '/'], "_"); + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 2, Duration::from_secs(2)), + ); + + // An independent session holds both contended keys, so the spinners' try-locks + // return false on every iteration and they stay in the retry loop. + let holder = sibling_pool(&pool, 2); + let mut held_conn = holder.acquire().await.expect("holder connection"); + for repo in ["spin-a", "spin-b"] { + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(advisory_lock_key(&owner_slug, repo)) + .fetch_one(&mut *held_conn) + .await + .expect("holder try-lock"); + assert!(taken.0, "the holder must own {repo}'s key"); + } + + let mut spinners = Vec::new(); + for repo in ["spin-a", "spin-b"] { + let store = store.clone(); + spinners.push(tokio::spawn(async move { + store.acquire_write(owner, repo).await + })); + } + // Let both reach the spin (each has done at least one failed try-lock by now). + tokio::time::sleep(Duration::from_millis(500)).await; + + let started = std::time::Instant::now(); + let uncontended = tokio::time::timeout( + Duration::from_secs(10), + store.acquire_write(owner, "spin-free"), + ) + .await + .expect("the uncontended acquire must return, not hang"); + let elapsed = started.elapsed(); + let free_guard = uncontended.unwrap_or_else(|e| { + panic!( + "an UNCONTENDED acquire_write on a DIFFERENT repo must not be starved by \ + spinners holding the lock pool; got: {e}" + ) + }); + assert!( + elapsed < Duration::from_secs(2), + "the uncontended acquire must not queue behind the spinners for the pool \ + acquire timeout; took {elapsed:?}" + ); + free_guard.release(false).await; + + // The drop-and-retake cycle must still END in a real, exclusive lock: free + // spin-a's key and the spinner that was cycling connections must take it. + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(advisory_lock_key(&owner_slug, "spin-a")) + .execute(&mut *held_conn) + .await + .expect("release spin-a"); + let winner = tokio::time::timeout(Duration::from_secs(15), spinners.remove(0)) + .await + .expect("the spinner must finish once its key frees") + .expect("spinner task") + .expect("the spinner must acquire once the key frees"); + let probe = sibling_pool(&pool, 2); + assert!( + !lock_is_free_elsewhere(&probe, advisory_lock_key(&owner_slug, "spin-a")).await, + "the lock a spinner finally took must be observably held from another session" + ); + winner.release(false).await; + + for s in spinners { + s.abort(); + } + sqlx::query("SELECT pg_advisory_unlock_all()") + .execute(&mut *held_conn) + .await + .expect("release the remaining holder lock"); + } + + /// #173 F1, the property the fix rests on: returning a lock-pool connection that + /// holds NOTHING runs `after_release`'s `pg_advisory_unlock_all()`, which is a no-op + /// and must not disturb a lock held on a DIFFERENT connection of the same pool. + /// Session advisory locks are per connection, so this is by construction, but the + /// spin fix depends on it, so it is proven by execution rather than assumed. + #[sqlx::test] + async fn returning_an_unlocked_connection_does_not_clear_another_connections_lock( + pool: PgPool, + ) { + let owner = "did:key:z6MkNoOpUnlockAll"; + let repo = "noop-unlock"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + let probe = sibling_pool(&pool, 2); + let lock_pool = build_lock_pool(&pool, 4, Duration::from_secs(5)); + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + lock_pool.clone(), + ); + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + + // Churn the pool: check out and drop connections that hold no lock, exactly what + // a spinning acquire now does between attempts. Each return fires + // pg_advisory_unlock_all() on that connection. + for _ in 0..10 { + let mut conn = lock_pool.acquire().await.expect("churn checkout"); + let _: (i32,) = sqlx::query_as("SELECT 1") + .fetch_one(&mut *conn) + .await + .expect("churn query"); + drop(conn); + tokio::time::sleep(Duration::from_millis(10)).await; + } + + assert!( + !lock_is_free_elsewhere(&probe, key).await, + "a held write lock must survive other lock-pool connections being returned" + ); + guard.release(true).await; + assert!( + lock_is_free_elsewhere(&probe, key).await, + "release must still free the lock after the churn" + ); + } + + /// #173 F1: lock-pool exhaustion is a DISTINCT error the handler can shed as a 503, + /// not a generic git 500. Both directions: an exhausted pool downcasts to + /// [`LockPoolBusy`], and an unrelated failure (a rejected repo name) does not. + #[sqlx::test] + async fn lock_pool_exhaustion_is_a_distinct_downcastable_error(pool: PgPool) { + let owner = "did:key:z6MkBusyDowncast"; + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 1, Duration::from_secs(1)), + ); + let held = store + .acquire_write(owner, "busy-a") + .await + .expect("first acquire"); + + let err = match store.acquire_write(owner, "busy-b").await { + Ok(_) => panic!("an exhausted lock pool must error, not hand back a guard"), + Err(e) => e, + }; + assert!( + err.downcast_ref::().is_some(), + "lock-pool exhaustion must be downcastable so the handler sheds 503, got: {err}" + ); + + // MUST-NOT: an ordinary rejection is not a capacity signal. + let other = match store.acquire_write(owner, "../escape").await { + Ok(_) => panic!("a traversal repo name must be rejected"), + Err(e) => e, + }; + assert!( + other.downcast_ref::().is_none(), + "a validation failure must not masquerade as lock-pool capacity, got: {other}" + ); + + held.release(false).await; + } + /// Round trip: the lock is observably HELD between acquire and release, and /// observably FREE after. Both checks run from an independent session; from /// the holding session they would pass vacuously (session locks are diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 50145652..bbaf319c 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -14,13 +14,13 @@ use tokio::process::Command; /// the work they admitted, so admission is released only when that work is truly /// done — not the instant the handler future drops on a client disconnect. /// -/// A move-only wrapper: no methods beyond construction and `Drop`. The handler -/// MOVEs its permits in and keeps no copy (a retained copy would drop early and -/// release admission the moment the future is dropped, defeating the guard). It is -/// threaded into `drive_git_child`, whose [`KillGroupOnDrop`] moves it into the -/// detached reaper on disconnect, so both permits drop only after the process group -/// is confirmed reaped (`kill(-pgid,0)==ESRCH`) rather than while the group is still -/// alive holding PIDs past the concurrency cap (#174 P1-a, plain-spawn residual). +/// A drop-only wrapper: nothing here inspects what it holds. The handler MOVEs its +/// permits in and keeps no copy (a retained copy would drop early and release admission +/// the moment the future is dropped, defeating the guard). It is threaded into +/// `drive_git_child`, whose [`KillGroupOnDrop`] moves it into the detached reaper on +/// disconnect, so both permits drop only after the process group is confirmed reaped +/// (`kill(-pgid,0)==ESRCH`) rather than while the group is still alive holding PIDs past +/// the concurrency cap (#174 P1-a, plain-spawn residual). /// /// The `be0cdd6` path-scoped upload-pack walk already applies this discipline by /// moving its permits into the `spawn_blocking`; this generalizes it to the plain @@ -31,6 +31,8 @@ pub struct AdmissionGuard { // 'static` so the guard can move into the detached reaper task. _global: Option>, _caller: Option>, + // Any further work-scoped hold that must outlive the process group; see `with_hold`. + _hold: Option>, } impl AdmissionGuard { @@ -40,8 +42,24 @@ impl AdmissionGuard { Self { _global: Some(Box::new(global)), _caller: caller.map(|c| Box::new(c) as Box), + _hold: None, } } + + /// Attach a further hold that must not be released until the process group is + /// reaped, and ride it through the same seam as the permits. + /// + /// The push handler uses this for the repo WRITE LOCK (#173 F2). Its + /// `guard.release(..)` line is only reached if `receive_pack` returns, so on a client + /// disconnect the lock used to be freed by the dropped future while the detached + /// reaper was still giving the group its SIGTERM grace, admitting a second + /// `receive-pack` on the same repo. Carrying the lock here holds it until the group + /// is ESRCH-confirmed gone, which is the same invariant the timeout path already + /// keeps ("a caller releasing a write lock can't race them", `reap_group_on_timeout`). + pub fn with_hold(mut self, hold: impl Send + 'static) -> Self { + self._hold = Some(Box::new(hold)); + self + } } /// Handle `GET /:owner/:repo/info/refs?service=git-upload-pack` diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 648d7428..d4f4e63d 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -58,6 +58,32 @@ struct DbStartupStatus { next_retry_secs: AtomicU64, } +/// Hard ceiling on the advisory-lock pool's `max_connections`. +/// +/// `max_concurrent_git_pushes` is validated all the way up to 1_048_576, and the lock +/// pool used to derive its size straight from that knob, so raising the push cap +/// silently raised the node's Postgres connection ceiling with no CLI error and no +/// relation to the server's own `max_connections` (#173 F4). The node's total budget is +/// now bounded: `db_max_connections` (default 20) + at most this. +const LOCK_POOL_MAX_CONNECTIONS: u32 = 64; + +/// Connections the lock pool keeps above the push cap. Covers the three non-push +/// `acquire_write` callers (`api/issues.rs` x2, `api/pulls.rs`), which hold no +/// concurrency permit, so a push never queues here for a connection where it did not +/// before. +const LOCK_POOL_PUSH_HEADROOM: u8 = 8; + +/// Size the advisory-lock pool for a given push cap: the cap plus +/// [`LOCK_POOL_PUSH_HEADROOM`], clamped to [`LOCK_POOL_MAX_CONNECTIONS`]. Past the +/// clamp a push may wait for a lock-pool connection, which is a bounded wait that sheds +/// a clean 503 (see `LockPoolBusy`), not an unbounded hang. +fn lock_pool_size(max_concurrent_git_pushes: usize) -> u32 { + u32::try_from(max_concurrent_git_pushes) + .unwrap_or(u32::MAX) + .saturating_add(u32::from(LOCK_POOL_PUSH_HEADROOM)) + .min(LOCK_POOL_MAX_CONNECTIONS) +} + #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() @@ -282,14 +308,11 @@ async fn main() -> Result<()> { // Repo write locks run on their own pool, never the main query pool: each // push holds its connection for the whole receive-pack, and // db_max_connections (20) is below max_concurrent_git_pushes (32), so sharing - // would starve every other query under a push burst. Headroom above the push - // cap keeps a push from ever queueing here for a connection where it did not - // before. See build_lock_pool for the cancellation semantics (#173). + // would starve every other query under a push burst. See build_lock_pool for + // the cancellation semantics (#173). let lock_pool = git::repo_store::build_lock_pool( db.pool(), - u32::try_from(config.max_concurrent_git_pushes) - .unwrap_or(u32::MAX) - .saturating_add(8), + lock_pool_size(config.max_concurrent_git_pushes), std::time::Duration::from_secs(config.db_acquire_timeout_secs), ); let repo_store = git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, lock_pool); @@ -1139,6 +1162,36 @@ fn load_or_create_keypair(config: &Config) -> Result { } } +#[cfg(test)] +mod lock_pool_sizing_tests { + use super::{lock_pool_size, LOCK_POOL_MAX_CONNECTIONS, LOCK_POOL_PUSH_HEADROOM}; + + /// The default push cap gets its cap plus headroom, so no push ever queues for a + /// lock-pool connection where it did not before. + #[test] + fn default_push_cap_gets_headroom_over_the_cap() { + assert_eq!(lock_pool_size(32), 32 + u32::from(LOCK_POOL_PUSH_HEADROOM)); + assert_eq!(lock_pool_size(1), 1 + u32::from(LOCK_POOL_PUSH_HEADROOM)); + } + + /// #173 F4: `max_concurrent_git_pushes` is validated all the way to 1_048_576, so an + /// operator raising it used to raise the node's Postgres connection ceiling with it, + /// silently and without bound. The lock pool is CLAMPED instead. + #[test] + fn an_oversized_push_cap_is_clamped_not_propagated() { + assert_eq!(lock_pool_size(1_048_576), LOCK_POOL_MAX_CONNECTIONS); + assert_eq!(lock_pool_size(usize::MAX), LOCK_POOL_MAX_CONNECTIONS); + // The largest cap that still fits under the clamp keeps its full headroom. + let widest = (LOCK_POOL_MAX_CONNECTIONS - u32::from(LOCK_POOL_PUSH_HEADROOM)) as usize; + assert_eq!(lock_pool_size(widest), LOCK_POOL_MAX_CONNECTIONS); + assert_eq!( + lock_pool_size(widest - 1), + LOCK_POOL_MAX_CONNECTIONS - 1, + "values below the clamp must not be rounded up to it" + ); + } +} + #[cfg(test)] mod gossip_ssrf_tests { use super::ping_peer_health; diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 47428248..303aed00 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -9995,6 +9995,198 @@ mod tests { "the key is released once the task is clean" ); } + + /// Wait for the tail's atomic check-and-clear to consume the pending dirty + /// bit (`Some(true)` -> `Some(false)`), which is the exact instant the task + /// enters `requeue_refresh_state`'s retry window. Deterministic, so the + /// coalescing push below lands INSIDE that window rather than on a sleep + /// guess. `None` means the key is already gone (the task exited), which the + /// caller reports as its own failure. + async fn wait_for_refresh_window( + inflight: &crate::state::EncryptInflight, + repo_id: &str, + ) -> bool { + for _ in 0..5_000 { + match inflight.dirty(repo_id) { + Some(false) => return true, + None => return false, + Some(true) => {} + } + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + false + } + + /// SCENARIO 7 (#173 F3, RED-before/GREEN-after). A push that coalesces WHILE + /// the re-read is retrying must not be thrown away when that re-read finally + /// gives up. `requeue_or_release` returning true cleared the dirty bit without + /// marking the guard released, so a `break` on `Failed` let + /// `EncryptInflightGuard::drop` remove the key outright and push C's pass was + /// never attempted, a silent drop with no reconciliation sweep behind it. + /// + /// Exactly `REQUEUE_REREAD_MAX_ATTEMPTS` injected repo-read faults, so the + /// first refresh exhausts its budget and the DB is healthy for the next one. + /// Push C coalesces inside that window. RED with `Failed => break`: obj_c is + /// never pinned. GREEN when `Failed` falls through and lets the tail decide. + #[sqlx::test] + async fn u2_failed_reread_keeps_a_push_that_coalesced_during_the_window(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-window"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj_a = commit(&git_repo.path, "a.txt", "one\n"); + let obj_c = commit(&git_repo.path, "c.txt", "three\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Exactly the bound: the FIRST refresh burns all three attempts and gives + // up; every later refresh sees a healthy DB. + requeue_faults::inject(&repo.id, 3, 0); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces while A is in flight" + ); + + // Push C lands during the retry window, after the tail already consumed + // push B's dirty bit. + let inflight = state.encrypt_inflight.clone(); + let repo_id = repo.id.clone(); + let coalesced = tokio::spawn(async move { + if !wait_for_refresh_window(&inflight, &repo_id).await { + return false; + } + inflight.try_begin(&repo_id).is_none() + }); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj_a.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + coalesced.await.expect("coalescing task"), + "push C must have coalesced inside the retry window for this test to \ + mean anything" + ); + assert!( + state.db.is_pinned(&obj_c).await.unwrap(), + "the push that coalesced during the retry window must still get a pass \ + once the DB recovers (RED with `Failed => break`: the dirty bit was \ + already consumed, so the pass was dropped with nothing to re-derive it)" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits" + ); + } + + /// SCENARIO 8 (#173 F3, the sustained-outage guard on the fall-through). Falling + /// through on `Failed` means `requeue_or_release` runs again, so a DB that never + /// recovers must still TERMINATE rather than spin. It does: an extra lap only + /// happens when a push actually coalesced, and each lap pays a full bounded + /// re-read (3 attempts with backoff). One coalescing push during the window buys + /// exactly one extra lap: 6 repo-read attempts, then exit. + #[sqlx::test] + async fn u2_sustained_failure_with_a_coalesce_terminates_after_one_more_lap( + pool: PgPool, + ) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-sustained-window"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj_a = commit(&git_repo.path, "a.txt", "one\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // The outage never clears. + requeue_faults::inject(&repo.id, 10_000, 0); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + let inflight = state.encrypt_inflight.clone(); + let repo_id = repo.id.clone(); + let coalesced = tokio::spawn(async move { + if !wait_for_refresh_window(&inflight, &repo_id).await { + return false; + } + inflight.try_begin(&repo_id).is_none() + }); + + // The watchdog is the real assertion: a fall-through that re-spins without + // the dirty gate would never return here. + tokio::time::timeout( + std::time::Duration::from_secs(60), + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj_a.clone()], + Some(vec![]), + HashSet::new(), + ), + ) + .await + .expect( + "the task must terminate under a sustained outage; a fall-through that \ + does not gate on the dirty bit spins forever", + ); + + assert!( + coalesced.await.expect("coalescing task"), + "push C must have coalesced inside the retry window" + ); + assert_eq!( + requeue_faults::counters(&repo.id).repo_read_attempts, + 6, + "one coalescing push buys exactly one more bounded re-read lap \ + (3 + 3 attempts), never an unbounded retry" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released on the give-up path" + ); + } } } } From ef9b7165cf0796c8e23e482ea3ecf935e5af6a25 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:29:01 -0500 Subject: [PATCH 21/77] fix(node): keep the incompleteness marker until a source is really recorded, and let the sweep retry Four follow-ups from an adversarial pass over the previous commits. The marker clear ran unconditionally inside `record_pin_source`'s transaction, but its INSERT is `ON CONFLICT DO NOTHING` guarded by the cap, so it records nothing when the repo is already a source. That is the common case, not a rare one: the pin path calls `record_pin_source` for every already-pinned object, and on a requeue pass that is the whole-repo enumeration. So a re-push from the repo that was already recorded cleared a marker set by a different repo's failure and the public copy went back to 404ing. The clear is now gated on the insert actually adding a row. A genuine record from a third repo still clears the marker, which needs a per-(oid, repo) table to close and is documented rather than papered over. The sweep advanced its cursor before every skip, which stops a wedge but made a transient skip permanent: once the walk reached the end of the table the cursor parked there and no later boot read another row. On a Tigris-backed node the common skip is "repo is not on local disk", so the sweep would repair whatever happened to be warm at boot and then never run again, and the new advertise filter turned those rows from "advertised but 404" into "never advertised, never repaired". Repairs now report whether a skip was retryable or terminal, and a run that skipped anything retryable rewinds the cursor once, after the walk, so the next run re-walks. A permanently unrepairable row is terminal and does not trigger a rewind, so this cannot become a hot loop. The sweep also reached its repo paths through the raw join helper, skipping the three-layer traversal barrier `RepoStore::local_path` applies. A test with a repo row named `../../escapee` showed it reading and repairing bytes from outside `repos_dir`. That barrier is now extracted and shared, so the two callers cannot drift. And the repair's synchronous `git cat-file` moved under `spawn_blocking`, matching the existing treatment of the withheld walk, so a wedged read no longer parks a worker thread for the full service timeout. Finally the resolver's two source-set queries ran ahead of the work-budget peek that exists to shed a spent caller before doing work. The peek now runs first. A throttled caller whose set turns out complete gets 429 where it used to get 404, which is the honest answer since its search never ran, and it removes an oracle that let a throttled caller tell a complete source set from an incomplete one. --- crates/gitlawb-node/src/api/ipfs.rs | 88 +++-- crates/gitlawb-node/src/db/mod.rs | 74 ++-- crates/gitlawb-node/src/git/repo_store.rs | 79 ++-- crates/gitlawb-node/src/ipfs_pin.rs | 153 ++++++-- crates/gitlawb-node/src/test_support.rs | 428 ++++++++++++++++++++++ 5 files changed, 711 insertions(+), 111 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index ba669027..e3116973 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -340,38 +340,52 @@ pub async fn get_by_cid( // Only a set with NONE of these three signals is treated as complete (every // recorded source was just tried), so it skips the scan and lets the tail 404, and // ordinary denials never fan out to O(repos) (INV-10 / F3). Both extra queries run - // only on a provenance MISS (we return above on Served), so neither costs the serve - // path, and the fallback is not an authorization bypass: the scan gates every repo + // only on a provenance MISS (we return above on Served) by a caller that still has + // work budget, so neither costs the serve path nor a shed caller, and the fallback + // is not an authorization bypass: the scan gates every repo // through the SAME per-caller gate, so a caller who may not read the object is // still denied. - let needs_scan = sources.is_empty() - || state + // + // F3 (#173, INV-10/INV-15): peek the per-IP WORK-budget limiter WITHOUT + // consuming a token so an already-throttled source is shed BEFORE the + // O(repos) preload; the consuming per-probe charge inside gate_and_serve is + // left UNCHANGED (it is load-bearing for the across-request bound), so this + // adds no double-charge. This peeks `ipfs_work_rate_limiter`, the SAME bucket + // the per-probe charge below debits — NOT the route limiter (`ipfs_rate_limiter`, + // charged once per request by the middleware): peeking the route bucket here + // would re-shed a request the route already admitted (R6, U5). + // + // The peek runs BEFORE the two marker queries (#173 round 11, F5): shedding is + // the whole point of a peek, so a spent-budget caller should not pay two + // lookups per request first. It stays AFTER the provenance walk, so no caller + // who could have been served is shed. The one caller this moves: a spent-budget + // caller whose source set turns out COMPLETE now takes the 429 tail instead of + // the 404 tail. That is the honest answer (its search never ran), and it drops + // an oracle, since the old order let a throttled caller tell a complete source + // set from an incomplete one by 404 vs 429. + if let Some(key) = + crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) + { + if state.ipfs_work_rate_limiter.is_throttled(&key).await { + throttled = true; + continue; + } + } + let needs_scan = sources.is_empty() || { + #[cfg(test)] + bump_marker_queries(); + state .db .pin_sources_at_cap(sha256_hex) .await .map_err(AppError::Internal)? - || state - .db - .pin_sources_incomplete(sha256_hex) - .await - .map_err(AppError::Internal)?; + || state + .db + .pin_sources_incomplete(sha256_hex) + .await + .map_err(AppError::Internal)? + }; if needs_scan { - // F3 (#173, INV-10/INV-15): peek the per-IP WORK-budget limiter WITHOUT - // consuming a token so an already-throttled source is shed BEFORE the - // O(repos) preload; the consuming per-probe charge inside gate_and_serve is - // left UNCHANGED (it is load-bearing for the across-request bound), so this - // adds no double-charge. This peeks `ipfs_work_rate_limiter`, the SAME bucket - // the per-probe charge below debits — NOT the route limiter (`ipfs_rate_limiter`, - // charged once per request by the middleware): peeking the route bucket here - // would re-shed a request the route already admitted (R6, U5). - if let Some(key) = - crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) - { - if state.ipfs_work_rate_limiter.is_throttled(&key).await { - throttled = true; - continue; - } - } // Load the scan context once, lazily (shared across oid candidates). if scan_ctx.is_none() { #[cfg(test)] @@ -910,6 +924,30 @@ fn bump_preload_queries() { PRELOAD_QUERIES.with(|c| c.set(c.get() + 1)); } +// Test-only cost counter (F5, #173 round 11): how many times the fallback gate ran the +// `pin_sources_at_cap` / `pin_sources_incomplete` pair. The work-budget peek sits ahead +// of them, so an already-throttled caller leaves this at 0; putting the peek back after +// the pair turns that assertion red. Same thread_local discipline as the preload counter. +#[cfg(test)] +thread_local! { + static MARKER_QUERIES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_marker_queries() { + MARKER_QUERIES.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn marker_queries() -> usize { + MARKER_QUERIES.with(|c| c.get()) +} + +#[cfg(test)] +fn bump_marker_queries() { + MARKER_QUERIES.with(|c| c.set(c.get() + 1)); +} + // Test-only INV-10 cost counter (F6, U6/U7): how many times the serve path withheld an // object because it exceeded `ipfs_max_served_object_bytes`. The bounded read must reject // an oversized object rather than buffer it on the worker; the counter is the both-ways diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 4b107da3..75f279d7 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -2488,17 +2488,26 @@ impl Db { /// keeping the first-pinner), so the INV-10 bound on serve-time work holds at /// `O(MAX_PIN_SOURCES + 1)` regardless of a table overshoot. /// - /// A successful record also CLEARS the `pin_sources_incomplete` marker for the - /// object, in the SAME transaction as the insert (U3, #173), so the clear cannot - /// drift across the four call sites or land without the row it describes. The - /// marker is per-object, not per-(object, repo): a record from repo B clears a - /// marker set by a failed record from repo A, which can re-hide A's hole until A - /// pushes again. That is the deliberate cost of a single boolean, and it fails in - /// the safe direction relative to today (the marker only ever ADDS the fallback, - /// never removes a source the resolver already tries). + /// A record that ACTUALLY ADDS a source row also CLEARS the + /// `pin_sources_incomplete` marker for the object, in the SAME transaction as the + /// insert (U3, #173), so the clear cannot drift across the four call sites or land + /// without the row it describes. + /// + /// The clear is gated on `rows_affected() > 0` because the INSERT is a no-op in two + /// ordinary cases: the `(oid, repo)` pair already exists (`ON CONFLICT DO NOTHING`) + /// and the source set is at cap (the count guard). The skip path calls this for + /// EVERY already-pinned object, and on a requeue pass that list is the whole-repo + /// enumeration, so an unconditional clear meant the next coalesced push from a repo + /// already in the set wiped the marker for every object in the repo without + /// recording anything (round 11 regression). The residual, which the gate does not + /// close: the marker is per-object, not per-(object, repo), so a GENUINE record from + /// a third repo C still clears a marker that repo A's failed record set. That is the + /// deliberate cost of a single boolean; closing it needs a per-(oid, repo) marker + /// table, and it fails in the safe direction (the marker only ever ADDS the scan + /// fallback, never removes a source the resolver already tries). pub async fn record_pin_source(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { let mut tx = self.pool.begin().await?; - sqlx::query( + let inserted = sqlx::query( "INSERT INTO pin_repo_sources (sha256_hex, repo_id) SELECT $1, $2 WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 @@ -2508,14 +2517,17 @@ impl Db { .bind(repo_id) .bind(MAX_PIN_SOURCES) .execute(&mut *tx) - .await?; - sqlx::query( - "UPDATE pinned_cids SET pin_sources_incomplete = FALSE - WHERE sha256_hex = $1 AND pin_sources_incomplete", - ) - .bind(sha256_hex) - .execute(&mut *tx) - .await?; + .await? + .rows_affected(); + if inserted > 0 { + sqlx::query( + "UPDATE pinned_cids SET pin_sources_incomplete = FALSE + WHERE sha256_hex = $1 AND pin_sources_incomplete", + ) + .bind(sha256_hex) + .execute(&mut *tx) + .await?; + } tx.commit().await?; Ok(()) } @@ -2526,6 +2538,13 @@ impl Db { /// source set that is silently missing its own first pinner. One transaction /// removes that window entirely: either both rows land or neither does, and a /// total failure leaves the object unpinned so the next push retries it. + /// + /// The marker clear carries the same `rows_affected` gate as `record_pin_source`. + /// It is not load-bearing here: this path runs only when `is_pinned` said no row + /// exists, and `mark_pin_sources_incomplete` is a no-op without a `pinned_cids` row, + /// so there is no marker to wrongly clear. The gate is kept for the one window that + /// is not covered by that argument, a concurrent pinner landing the row between the + /// `is_pinned` check and this upsert, and so the two clears cannot drift apart. pub async fn record_pinned_cid_with_source( &self, sha256_hex: &str, @@ -2545,7 +2564,7 @@ impl Db { .bind(repo_id) .execute(&mut *tx) .await?; - sqlx::query( + let inserted = sqlx::query( "INSERT INTO pin_repo_sources (sha256_hex, repo_id) SELECT $1, $2 WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 @@ -2555,14 +2574,17 @@ impl Db { .bind(repo_id) .bind(MAX_PIN_SOURCES) .execute(&mut *tx) - .await?; - sqlx::query( - "UPDATE pinned_cids SET pin_sources_incomplete = FALSE - WHERE sha256_hex = $1 AND pin_sources_incomplete", - ) - .bind(sha256_hex) - .execute(&mut *tx) - .await?; + .await? + .rows_affected(); + if inserted > 0 { + sqlx::query( + "UPDATE pinned_cids SET pin_sources_incomplete = FALSE + WHERE sha256_hex = $1 AND pin_sources_incomplete", + ) + .bind(sha256_hex) + .execute(&mut *tx) + .await?; + } tx.commit().await?; Ok(()) } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 226b50f2..ebdd7757 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -323,40 +323,61 @@ impl RepoStore { /// segment is rejected. This is the CodeQL-recognised barrier /// pattern for `rust/path-injection`. fn local_path(&self, owner_did: &str, repo_name: &str) -> Result<(String, PathBuf)> { - validate_path_components(owner_did, repo_name)?; - let owner_slug = owner_did.replace([':', '/'], "_"); - let local_path = self - .repos_dir - .join(&owner_slug) - .join(format!("{repo_name}.git")); - - if !local_path.starts_with(&self.repos_dir) { - anyhow::bail!( - "computed repo path escaped repos_dir: {}", - local_path.display() - ); - } + let local_path = validated_repo_disk_path(&self.repos_dir, owner_did, repo_name)?; + Ok((owner_slug, local_path)) + } +} - // Explicit component walk — sanitisation barrier that static analysers - // (CodeQL `rust/path-injection`) recognise. The path must be composed - // entirely of Normal segments after the root prefix; any ParentDir or - // CurDir component is a traversal attempt. - for component in local_path.components() { - use std::path::Component; - match component { - Component::Prefix(_) | Component::RootDir | Component::Normal(_) => {} - Component::ParentDir => { - anyhow::bail!("path contains parent-directory component"); - } - Component::CurDir => { - anyhow::bail!("path contains current-directory component"); - } +/// The three-layer validated form of `store::repo_disk_path`, with NO Tigris fetch and +/// no `RepoStore` (#173 round 11, F3). Extracted from `RepoStore::local_path` so a +/// second caller that must not pull a cold repo, the U4 legacy provider-CID sweep, gets +/// the same barrier instead of the raw join. `local_path` is now a thin wrapper over +/// this, so the two cannot drift. +/// +/// Three-layer defence against path traversal: +/// 1. Strict allowlist on `owner_did` and `repo_name` (no `..`, slashes, +/// null bytes, leading dots; length-bounded). +/// 2. The joined path must remain rooted at `repos_dir`. +/// 3. Every component of the joined path must be `Component::Normal` +/// (or the prefix/root from `repos_dir`); any `ParentDir`/`CurDir` +/// segment is rejected. This is the CodeQL-recognised barrier +/// pattern for `rust/path-injection`. +pub(crate) fn validated_repo_disk_path( + repos_dir: &Path, + owner_did: &str, + repo_name: &str, +) -> Result { + validate_path_components(owner_did, repo_name)?; + + let owner_slug = owner_did.replace([':', '/'], "_"); + let local_path = repos_dir.join(&owner_slug).join(format!("{repo_name}.git")); + + if !local_path.starts_with(repos_dir) { + anyhow::bail!( + "computed repo path escaped repos_dir: {}", + local_path.display() + ); + } + + // Explicit component walk — sanitisation barrier that static analysers + // (CodeQL `rust/path-injection`) recognise. The path must be composed + // entirely of Normal segments after the root prefix; any ParentDir or + // CurDir component is a traversal attempt. + for component in local_path.components() { + use std::path::Component; + match component { + Component::Prefix(_) | Component::RootDir | Component::Normal(_) => {} + Component::ParentDir => { + anyhow::bail!("path contains parent-directory component"); + } + Component::CurDir => { + anyhow::bail!("path contains current-directory component"); } } - - Ok((owner_slug, local_path)) } + + Ok(local_path) } /// Strict allowlist validator for `owner_did` and `repo_name`. diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 1f0cc65a..dd9b8545 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -66,37 +66,77 @@ async fn repair_legacy_provider_cid( git_timeout: Duration, sha: &str, db: &crate::db::Db, -) -> Result<()> { +) -> Result { let stored = match db.cid_for_oid(sha).await? { Some(c) => c, - None => return Ok(()), + None => return Ok(RepairOutcome::Settled), }; // Cost gate: a canonical raw CIDv1 key is already correct — never read bytes. if gitlawb_core::cid::is_raw_cidv1(&stored) { - return Ok(()); + return Ok(RepairOutcome::Settled); } // Legacy-codec row: read the object to recompute. Counted so a test can prove // the gate above spares non-legacy rows this read. #[cfg(test)] note_legacy_repair_read(); - let data = match crate::git::store::read_object_bounded(git_bin, repo_path, sha, git_timeout) { - Ok(Some((_ty, bytes))) => bytes, - // Bytes gone: the row stays withheld, never destructively rewritten. - Ok(None) => return Ok(()), + // `read_object_bounded` is SYNCHRONOUS `git cat-file`, and its budget is + // `git_service_timeout_secs` (600 by default), so running it inline parks a tokio + // worker for as long as git takes: one wedged read on the sweep's first pass at boot + // holds a worker for ten minutes, per legacy row. Push it to the blocking pool, the + // same shape `replication_withheld_set` uses in api/repos.rs (#173 round 11, F4). + // Both callers of this function are async, so neither changes shape. The read-counter + // increment above stays on THIS thread so the thread_local cost-gate assertion holds. + let read = { + let repo_path = repo_path.to_path_buf(); + let git_bin = git_bin.to_string(); + let sha = sha.to_string(); + tokio::task::spawn_blocking(move || { + crate::git::store::read_object_bounded(&git_bin, &repo_path, &sha, git_timeout) + }) + .await + }; + let data = match read { + Ok(Ok(Some((_ty, bytes)))) => bytes, + // Bytes gone: the row stays withheld, never destructively rewritten. Nothing a + // later pass changes, so this is a TERMINAL outcome for the sweep's re-walk gate. + Ok(Ok(None)) => return Ok(RepairOutcome::Settled), // A wedged/D-state `git cat-file` (timeout/infra): the repair is opportunistic // and best-effort, so skip it and return Ok so the pin task PROCEEDS to // requeue_or_release rather than hanging the coalescing key until process death // (grok F2, #173). A later re-push or the deferred sweep retries the repair. - Err(e) => { + Ok(Err(e)) => { tracing::warn!(sha = %sha, err = %e, "skipping legacy provider-CID repair: bounded object read failed"); - return Ok(()); + return Ok(RepairOutcome::Retryable); + } + // The blocking task panicked or was cancelled: same best-effort treatment, and + // worth another walk because it says nothing about the row itself. + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "skipping legacy provider-CID repair: object read task failed"); + return Ok(RepairOutcome::Retryable); } }; let raw = Cid::from_git_object_bytes(&data).to_string(); if raw == stored { - return Ok(()); + return Ok(RepairOutcome::Settled); } - db.repair_legacy_provider_cid(sha, &raw, &stored).await + db.repair_legacy_provider_cid(sha, &raw, &stored).await?; + Ok(RepairOutcome::Repaired) +} + +/// What one opportunistic repair did with a row, so the sweep can tell a skip a later +/// run could fix from one nothing will (U4 re-walk, #173 round 11). The push skip path +/// ignores the value: it repairs whatever the push happens to carry and a failure there +/// is already warn-only. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RepairOutcome { + /// Nothing to do, or nothing a re-walk would change: the stored key is already the + /// raw resolver key, the recomputed key matches it, or the object's bytes are gone. + Settled, + /// The bounded object read failed (a wedged `git cat-file`, an unreadable repo). + /// The bytes may be readable later, so the row is worth walking again. + Retryable, + /// The row's key was rewritten to the raw-content CID. + Repaired, } /// What one sweep pass (or a whole sweep run) did. `scanned` counts `pinned_cids` @@ -107,6 +147,12 @@ pub(crate) struct SweepStats { pub scanned: usize, pub repaired: usize, pub passes: usize, + /// Rows left unrepaired for a reason a LATER run could fix (the source repo is not + /// on this node's local disk, a DB read failed, a bounded object read failed). A + /// nonzero count is what makes the run rewind its cursor instead of parking it at + /// the end of the table forever. Rows that are unrepairable in principle (no + /// provenance, the repo row is gone, the bytes are gone) are NOT counted here. + pub retryable_skips: usize, } /// One bounded pass of the U4 sweep: read at most `batch` `pinned_cids` rows after @@ -131,6 +177,7 @@ async fn sweep_pass( let rows = db.pinned_cids_after(&cursor, batch).await?; let scanned = rows.len(); let mut repaired = 0usize; + let mut retryable_skips = 0usize; let mut last = cursor; for (sha, stored) in rows { @@ -149,16 +196,24 @@ async fn sweep_pass( Ok(s) => s, Err(e) => { tracing::warn!(sha = %sha, err = %e, "sweep: failed to read pin sources"); + // A DB read error says nothing about the row, so a later run retries it. + retryable_skips += 1; continue; } }; + // Whether this row ended the source walk repaired, and whether anything it hit + // along the way was a transient obstacle rather than a permanent one. + let mut row_repaired = false; + let mut row_retryable = false; for repo_id in sources { let repo = match db.get_repo_by_id(&repo_id).await { Ok(Some(r)) => r, - // The repo row is gone: a later source may still hold the bytes. + // The repo row is gone: a later source may still hold the bytes. A + // deleted repo does not come back, so this is not a retryable skip. Ok(None) => continue, Err(e) => { tracing::warn!(repo_id = %repo_id, err = %e, "sweep: failed to read repo"); + row_retryable = true; continue; } }; @@ -166,32 +221,50 @@ async fn sweep_pass( // The sweep is opportunistic background maintenance over every pinned row on // the node, so it must never pull a cold repo back from remote storage: that // would turn a repair pass into a bulk restore. A repo that is not on local - // disk simply reads no bytes here and stays withheld for a later pass or a - // re-push, which is the same non-destructive outcome as missing bytes. - let repo_path = - crate::git::store::repo_disk_path(repos_dir, &repo.owner_did, &repo.name); - if let Err(e) = - repair_legacy_provider_cid(&repo_path, git_bin, git_timeout, &sha, db).await - { - tracing::warn!(sha = %sha, err = %e, "sweep: legacy provider-CID repair failed"); + // disk simply reads no bytes here and stays withheld, but it IS a retryable + // skip: on a Tigris-backed node the repo is cold now and warm later, and + // without the re-walk that row would never be repaired by anything. + // The path goes through the repo store's VALIDATED resolver (allowlisted + // components, rooted at repos_dir, no ParentDir/CurDir segment), not the raw + // join: the sweep is a second caller of that path logic and gets the same + // barrier the acquire path has (#173 round 11, F3). It is the non-fetching + // variant, so the no-cold-pull property above is untouched. + let repo_path = match crate::git::repo_store::validated_repo_disk_path( + repos_dir, + &repo.owner_did, + &repo.name, + ) { + Ok(p) => p, + // An unsafe name is not something a later run fixes, so it is terminal. + Err(e) => { + tracing::warn!(repo_id = %repo_id, err = %e, "sweep: rejected unsafe repo path"); + continue; + } + }; + if !repo_path.is_dir() { + row_retryable = true; continue; } - // `repair_legacy_provider_cid` is best-effort and silent about which of its - // outcomes it took (bytes gone, read failed, rewritten), so read the key back - // to decide whether to stop trying sources. A no-op re-read on an - // already-repaired row is one indexed lookup. - match db.cid_for_oid(&sha).await { - Ok(Some(c)) if gitlawb_core::cid::is_raw_cidv1(&c) => { + match repair_legacy_provider_cid(&repo_path, git_bin, git_timeout, &sha, db).await { + Ok(RepairOutcome::Repaired) => { repaired += 1; + row_repaired = true; break; } - Ok(_) => continue, + // The bytes could not be read from this source right now: try the next + // source, and if none of them works, walk the row again on a later run. + Ok(RepairOutcome::Retryable) => row_retryable = true, + // Nothing to repair from this source and nothing a re-walk changes. + Ok(RepairOutcome::Settled) => {} Err(e) => { - tracing::warn!(sha = %sha, err = %e, "sweep: failed to re-read repaired key"); - continue; + tracing::warn!(sha = %sha, err = %e, "sweep: legacy provider-CID repair failed"); + row_retryable = true; } } } + if !row_repaired && row_retryable { + retryable_skips += 1; + } } db.set_pin_repair_cursor(&last).await?; @@ -199,6 +272,7 @@ async fn sweep_pass( scanned, repaired, passes: 1, + retryable_skips, }) } @@ -230,6 +304,16 @@ pub(crate) async fn sweep_legacy_provider_cids_once( /// its cursor every pass so a restart continues instead of rewinding. Errors reading /// or repairing an individual row are warn-and-skip; only a failure of the batch query /// or the cursor write ends the run, and a later run picks up from the stored cursor. +/// +/// A run that skipped at least one RETRYABLE row rewinds the cursor to the start of the +/// table on its way out (#173 round 11). Without that the cursor parked at the maximum +/// `sha256_hex` for good: every later boot read zero rows, so a row skipped for a +/// transient reason (its repo cold on a Tigris-backed node, a DB or object read error) +/// was skipped permanently, unadvertised and unresolvable with nothing left to fix it. +/// The rewind is a per-RUN decision made after the walk has already finished, never +/// mid-walk, so it cannot spin: the cost is one extra ordered scan on the next run, and +/// a row that is unrepairable in principle (bytes gone, provenance gone) does not count +/// as retryable, so a node holding one does not re-walk on every boot forever. pub(crate) async fn sweep_legacy_provider_cids( repos_dir: &std::path::Path, git_bin: &str, @@ -244,19 +328,26 @@ pub(crate) async fn sweep_legacy_provider_cids( Ok(p) => p, Err(e) => { tracing::warn!(err = %e, "legacy provider-CID sweep pass failed; stopping"); - return totals; + break; } }; totals.scanned += pass.scanned; totals.repaired += pass.repaired; + totals.retryable_skips += pass.retryable_skips; totals.passes += 1; // A short batch means the ordered walk reached the end of the table. Stop here // rather than after an extra empty pass, and do NOT sleep on the way out. if (pass.scanned as i64) < batch { - return totals; + break; } tokio::time::sleep(delay).await; } + if totals.retryable_skips > 0 { + if let Err(e) = db.set_pin_repair_cursor("").await { + tracing::warn!(err = %e, "failed to rewind the legacy provider-CID sweep cursor"); + } + } + totals } // Test-only cost-gate counter (R8, U7): how many times the opportunistic repair diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 303aed00..440a7cd1 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -2973,6 +2973,148 @@ mod tests { ); } + /// F5 (#173 round 11): the work-budget peek sheds an already-throttled caller BEFORE + /// the two marker queries, so a spent-budget source stops paying two lookups per + /// request for a scan it will never be allowed to run. The source set here is + /// non-empty and complete, which is the case that used to reach the queries anyway. + /// The counter is the both-ways guard: moving the peek back below the pair reads 1. + #[sqlx::test] + async fn ipfs_cid_throttled_caller_sheds_before_the_marker_queries(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + // One PRIVATE source, recorded cleanly: the set is non-empty, below cap and + // unmarked, so nothing but the peek can keep the request off the queries. + let fx = seed_cid_repos(&slug, &short, &["f5only"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("f5only.git"); + let mut repo = seed_repo(&owner_did, "f5only"); + repo.is_public = false; + state.db.create_repo(&repo).await.expect("seed private"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + state + .db + .record_pin_source(&fx.secret_oid, &repo.id) + .await + .expect("record source"); + + // Spend the caller's whole work budget before the request. + assert!( + state.ipfs_work_rate_limiter.check("9.9.9.9").await, + "the budget starts with room" + ); + + crate::api::ipfs::reset_marker_queries(); + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "a spent-budget caller is shed at the peek" + ); + assert!( + !body.contains("TOP SECRET"), + "the shed response must not leak the withheld object" + ); + assert_eq!( + crate::api::ipfs::marker_queries(), + 0, + "a shed caller pays neither marker query" + ); + } + + /// U3 scenario 6 (#173, regression): a record that inserts NOTHING must not clear + /// the marker. `record_pin_source` is called for EVERY already-pinned object on the + /// skip path, and on a requeue pass that is the whole-repo enumeration, so the next + /// coalesced push from a repo ALREADY in the source set re-runs the insert as a + /// no-op. Clearing on that no-op re-hides the hole a different repo's failed record + /// recorded: the public copy stops being scanned for and 404s again. The assertion + /// is the SERVE outcome, not the column, so it still bites if the resolver ever + /// stops consulting the marker. RED before the rows_affected gate (404); GREEN after. + #[sqlx::test] + async fn ipfs_cid_noop_record_must_not_clear_the_marker(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3npriv", "u3npub"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3npriv.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3npub.git"); + + // Repo A (private) is the first pinner AND is already recorded as a source, so a + // later record from A is a pure no-op insert. + let mut priv_repo = seed_repo(&owner_did, "u3npriv"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + state + .db + .record_pin_source(&fx.public_oid, &priv_repo.id) + .await + .expect("record the first pinner as a source"); + + // Repo B (public) holds the same object, but its source record never lands, so + // the node marks the set known-incomplete. + let pub_repo = seed_repo(&owner_did, "u3npub"); + state.db.create_repo(&pub_repo).await.expect("seed public"); + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &pub_bare, &fx.public_oid, &pub_repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "the exhausted record marked the set incomplete" + ); + + // A pushes again. The insert affects zero rows (A is already a source), so it + // recorded nothing and must not claim the set is complete. + repin_via_skip_branch(&state, &priv_bare, &fx.public_oid, &priv_repo.id).await; + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![priv_repo.id.clone()], + "the re-push really did add no source" + ); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a no-op record must not clear the marker: the public copy still has to serve" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + } + /// U3 scenario 4 (#173): the marker tracks the record's OUTCOME, not the attempt. /// An exhausted retry sets it; a first-attempt success never does. Without the /// second arm the first could be satisfied by marking unconditionally. @@ -4740,6 +4882,292 @@ mod tests { } } + /// U4 scenario 9 (#173, regression): a row skipped for a TRANSIENT reason is + /// retried by a later run. The sweep never pulls a cold repo back from remote + /// storage, so on a Tigris-backed node a repo that is not on local disk at boot + /// contributes nothing to the pass. With the cursor parked at the end of the table + /// that row was skipped FOREVER: every later boot read zero rows and the row stayed + /// unadvertised and unresolvable with nothing left to repair it. Here the repo is + /// off disk for the first run and back for the second, so only a re-walk repairs it. + /// RED before the transient-skip cursor reset (the second run scans nothing). + #[sqlx::test] + async fn sweep_rewalks_after_a_transient_skip(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["coldsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("coldsrc.git"); + let repo = seed_repo(&owner_did, "coldsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let (raw_cid, _provider) = + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + // The repo is COLD: its provenance resolves, but the bytes are not on this + // node's disk right now, exactly the state the sweep refuses to fix by pulling. + let stashed_away = bare.with_extension("git.away"); + let _ = std::fs::remove_dir_all(&stashed_away); + std::fs::rename(&bare, &stashed_away).expect("take the repo off local disk"); + + let first = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the first run terminates"); + assert_eq!( + (first.scanned, first.repaired), + (1, 0), + "the cold repo's row is walked but cannot be repaired yet" + ); + + // The repo is warm again (a later boot, a fetch, an operator restore). + std::fs::rename(&stashed_away, &bare).expect("put the repo back on local disk"); + + let second = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the second run terminates"); + assert_eq!( + second.repaired, 1, + "a later run re-walks the transiently skipped row and repairs it" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the row now carries the raw-content resolver key" + ); + assert!( + state + .db + .list_pinned_cids() + .await + .unwrap() + .iter() + .any(|r| r.cid == raw_cid), + "the repaired row is advertised again" + ); + } + + /// U4 scenario 10 (#173, the other arm of scenario 9): a PERMANENTLY unrepairable + /// row must not make the sweep re-walk forever. Bytes that are genuinely gone are a + /// terminal skip, so the cursor stays parked and a later run reads nothing. Without + /// that split the transient-skip reset of scenario 9 turns every boot on such a node + /// into a full table walk. Both runs are timeout-bounded, so a hot loop FAILS here. + #[sqlx::test] + async fn sweep_does_not_rewalk_for_a_terminal_skip(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + // The repo IS on local disk; the object's bytes are not in it and never will be. + let _fx = seed_cid_repos(&slug, &short, &["termsrc"]); + let repo = seed_repo(&owner_did, "termsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let phantom_oid = "e".repeat(64); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&phantom_oid) + .bind(legacy_dagpb_cid(&raw_cid)) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + let first = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the first run terminates"); + assert_eq!( + (first.scanned, first.repaired), + (1, 0), + "the row is walked and cannot be repaired" + ); + + let second = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the second run terminates"); + assert_eq!( + (second.scanned, second.repaired, second.passes), + (0, 0, 1), + "a terminal skip leaves the cursor parked: the next run re-reads nothing" + ); + } + + /// U4 scenario 11 (#173, path barrier): the sweep resolves a source repo's disk path + /// through the SAME validated logic the repo store uses, so a repo row whose name + /// carries `..` reads nothing. Names are validated at creation today, so this is a + /// defence-in-depth barrier on a second caller of the raw path helper rather than a + /// live exploit. The escapee repo really does hold the object's bytes, so before the + /// barrier the sweep happily read them from outside `repos_dir` and repaired the row. + /// RED before routing through the validated path (repaired 1). + #[sqlx::test] + async fn sweep_refuses_a_source_path_that_escapes_repos_dir(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + // The bytes live at /tmp/{slug}/escapee.git, OUTSIDE the repos_dir below. + let fx = seed_cid_repos(&slug, &short, &["escapee"]); + let escapee_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("escapee.git"); + let repos_dir = std::path::PathBuf::from("/tmp").join(&slug).join("root"); + std::fs::create_dir_all(repos_dir.join(&slug)).expect("create the repos_dir tree"); + + // A repo row whose name walks back out of repos_dir: repos_dir/{slug}/../../escapee.git + let mut repo = seed_repo(&owner_did, "../../escapee"); + repo.disk_path = escapee_bare.display().to_string(); + state.db.create_repo(&repo).await.expect("seed repo"); + let (_raw_cid, provider_cid) = + seed_legacy_pin(&pool, &escapee_bare, &fx.public_oid, Some(&repo.id)).await; + assert!( + crate::git::store::repo_disk_path(&repos_dir, &owner_did, &repo.name).exists(), + "the unvalidated helper really does resolve to the escapee repo" + ); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + &repos_dir, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates"); + + assert_eq!( + stats.repaired, 0, + "a repo path that escapes repos_dir must never be read" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "the row is untouched because its bytes were never read" + ); + } + + /// U4 scenario 12 (#173, F4): the repair's object read is SYNCHRONOUS `git cat-file`, + /// so running it inline parks the async worker for as long as git takes, up to the + /// whole `git_service_timeout_secs` budget on a wedged read, and the sweep does this + /// per legacy row starting at boot. A slow git stand-in makes that observable: a + /// concurrent 20ms ticker cannot tick at all while the only worker thread is blocked, + /// and ticks freely once the read is on the blocking pool. RED before the + /// `spawn_blocking` (0 ticks). + #[sqlx::test] + async fn repair_object_read_does_not_block_the_async_worker(pool: PgPool) { + use gitlawb_core::identity::Keypair; + use std::sync::atomic::{AtomicUsize, Ordering}; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["slowsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("slowsrc.git"); + let repo = seed_repo(&owner_did, "slowsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + // A git that takes 300ms per invocation (the read makes two: type, then content). + let slow_git = std::env::temp_dir().join(format!("gl-slow-git-{short}")); + std::fs::write(&slow_git, "#!/bin/sh\nsleep 0.3\nexec git \"$@\"\n").expect("write shim"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&slow_git, std::fs::Permissions::from_mode(0o755)) + .expect("chmod shim"); + } + + let ticks = std::sync::Arc::new(AtomicUsize::new(0)); + let ticker = { + let ticks = ticks.clone(); + tokio::spawn(async move { + loop { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + ticks.fetch_add(1, Ordering::Relaxed); + } + }) + }; + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + slow_git.to_str().unwrap(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ) + .await; + ticker.abort(); + + assert_eq!(stats.repaired, 1, "the slow git still repairs the row"); + assert!( + ticks.load(Ordering::Relaxed) >= 5, + "the runtime kept running other tasks during the blocking git read (ticks: {})", + ticks.load(Ordering::Relaxed) + ); + } + /// U4 scenario 8 (#173, degenerate states): an empty `pinned_cids` table and a /// table with zero legacy rows both complete cleanly, with no repair and no read. #[sqlx::test] From d85416127bf60ff0b176f1e5e048039699cef9f4 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:14:55 -0500 Subject: [PATCH 22/77] fix(node): shed 503 from the issue and PR write paths when the lock pool is exhausted The lock-pool work gave `acquire_write` a distinct exhaustion error and mapped it to 503 with Retry-After, but only on the push handler. The three other callers, both issue writes and the PR merge, still turned it into a generic git error and returned 500. Those are exactly the callers that hold no concurrency permit, so they are the likeliest to meet an exhausted pool in the first place. All three now go through the push handler's helper rather than repeating the mapping, so the log level and the message cannot drift between the four sites. Each has a test that occupies the single connection of a one-slot pool with a guard on a different repo, so the failure is pool capacity rather than advisory-lock contention, and asserts status 503 plus retry-after through IntoResponse. Reverting the mapping at one site turns only that site's test red. Worth knowing for review, and left alone deliberately: close_issue takes the write guard before its author check, so this shed is reachable by any authenticated caller whether or not they could close the issue. That ordering predates this change and reordering the handler is not this commit's business, but it does mean the 503 discloses that the repo's write pool is busy. --- crates/gitlawb-node/src/api/issues.rs | 174 +++++++++++++++++++++++++- crates/gitlawb-node/src/api/pulls.rs | 118 ++++++++++++++++- crates/gitlawb-node/src/api/repos.rs | 6 +- 3 files changed, 294 insertions(+), 4 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 17acf9af..0eacfa72 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -61,11 +61,14 @@ pub async fn create_issue( let json_str = serde_json::to_string(&issue) .map_err(|e| AppError::BadRequest(format!("serialization error: {e}")))?; + // Shed 503 + Retry-After on an exhausted write-lock POOL instead of a generic + // git 500 (#173 F1). This path holds no admission permit, so it reaches the pool + // unthrottled; reuse the push handler's mapping so the two cannot drift. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) .await - .map_err(|e| AppError::Git(e.to_string()))?; + .map_err(|e| crate::api::repos::acquire_write_app_error(&e, &repo))?; let disk_path = guard.path().to_path_buf(); let create_result = git_issues::create_issue(&disk_path, &issue_id, &json_str); @@ -229,11 +232,13 @@ pub async fn close_issue( .await? .ok_or_else(|| AppError::RepoNotFound(format!("{owner}/{repo}")))?; + // Same capacity shed as create_issue above (#173 F1): an exhausted write-lock + // pool is a 503 + Retry-After, not a 500 git error. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) .await - .map_err(|e| AppError::Git(e.to_string()))?; + .map_err(|e| crate::api::repos::acquire_write_app_error(&e, &repo))?; let disk_path = guard.path().to_path_buf(); // Owner OR issue author may close. The author lives in the issue's git-JSON @@ -279,3 +284,168 @@ pub async fn close_issue( Ok(Json(issue)) } + +/// #173 F1 follow-up: the two issue write paths reach `acquire_write` holding NO +/// admission permit (unlike the push handler, which is capped by the git-push +/// semaphore), so they are the callers most likely to meet an exhausted write-lock +/// POOL under load. An exhausted pool is a capacity signal, so both must shed +/// 503 + Retry-After (`AppError::Overloaded`) the way the push handler does, not +/// report the generic 500 git error that says nothing about retrying. +#[cfg(test)] +mod lock_pool_shed_tests { + use super::*; + use axum::response::IntoResponse; + use sqlx::PgPool; + + fn seed_repo(owner_did: &str, name: &str) -> crate::db::RepoRecord { + let now = Utc::now(); + crate::db::RepoRecord { + id: Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner_did.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/tmp/{name}"), + forked_from: None, + machine_id: None, + } + } + + /// State whose repo store draws write locks from a ONE-connection pool with a + /// short checkout timeout, so a single held guard exhausts it promptly rather + /// than at the pool default. + async fn one_connection_lock_pool_state(pool: &PgPool) -> AppState { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.repo_store = crate::git::repo_store::RepoStore::new( + std::path::PathBuf::from("/tmp/gitlawb-issues-lockpool"), + None, + crate::git::repo_store::build_lock_pool(pool, 1, std::time::Duration::from_secs(1)), + ); + state + } + + /// The shed must be a real 503 carrying Retry-After, not just an internal enum + /// variant: assert on the rendered response so a remapping of `Overloaded` is + /// caught here too. + fn assert_sheds_503_with_retry_after(err: AppError, what: &str) { + let resp = err.into_response(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "{what}: an exhausted write-lock pool must shed 503, not a 500 git error" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .map(|v| v.to_str().unwrap()), + Some("1"), + "{what}: a capacity shed must tell the client when to retry" + ); + } + + /// RED-before/GREEN-after for `create_issue`. Both directions: the shed while the + /// only lock-pool connection is held by a guard on a DIFFERENT repo (so this is + /// pool capacity, not advisory-lock contention on this repo), and the must-not + /// case once that connection is back. + #[sqlx::test] + async fn create_issue_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { + let owner = "did:key:zISSUECREATELOCKPOOLAAAAAAAAAAAAAAAAAAAA"; + let state = one_connection_lock_pool_state(&pool).await; + state + .db + .create_repo(&seed_repo(owner, "lp-create")) + .await + .expect("seed repo"); + + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = create_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-create".to_string())), + Json(CreateIssueRequest { + title: "t".to_string(), + body: None, + signed_payload: None, + }), + ) + .await; + let err = shed.expect_err("an exhausted lock pool must fail the call"); + assert_sheds_503_with_retry_after(err, "create_issue"); + + // MUST-NOT: with the pool free again the call is not shed as capacity (it + // fails later on the nonexistent on-disk repo, which is a git 500). + held.release(false).await; + let admitted = create_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-create".to_string())), + Json(CreateIssueRequest { + title: "t".to_string(), + body: None, + signed_payload: None, + }), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, create_issue must not be shed as capacity; got {:?}", + admitted.err() + ); + } + + /// RED-before/GREEN-after for `close_issue`, same two directions. + #[sqlx::test] + async fn close_issue_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { + let owner = "did:key:zISSUECLOSELOCKPOOLBBBBBBBBBBBBBBBBBBBBB"; + let state = one_connection_lock_pool_state(&pool).await; + state + .db + .create_repo(&seed_repo(owner, "lp-close")) + .await + .expect("seed repo"); + + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = close_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path(( + owner.to_string(), + "lp-close".to_string(), + "deadbeef".to_string(), + )), + ) + .await; + let err = shed.expect_err("an exhausted lock pool must fail the call"); + assert_sheds_503_with_retry_after(err, "close_issue"); + + held.release(false).await; + let admitted = close_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path(( + owner.to_string(), + "lp-close".to_string(), + "deadbeef".to_string(), + )), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, close_issue must not be shed as capacity; got {:?}", + admitted.err() + ); + } +} diff --git a/crates/gitlawb-node/src/api/pulls.rs b/crates/gitlawb-node/src/api/pulls.rs index 26be6109..6255ef24 100644 --- a/crates/gitlawb-node/src/api/pulls.rs +++ b/crates/gitlawb-node/src/api/pulls.rs @@ -209,11 +209,14 @@ pub async fn merge_pr( return Err(AppError::BadRequest(format!("PR is already {}", pr.status))); } + // Shed 503 + Retry-After on an exhausted write-lock POOL instead of a generic + // git 500 (#173 F1). Merging holds no admission permit, so it reaches the pool + // unthrottled; reuse the push handler's mapping so the two cannot drift. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) .await - .map_err(|e| AppError::Git(e.to_string()))?; + .map_err(|e| crate::api::repos::acquire_write_app_error(&e, &name))?; let disk_path = guard.path().to_path_buf(); let merger_did = auth.0; let merge_result = store::merge_branch( @@ -424,3 +427,116 @@ pub async fn list_comments( let comments = state.db.list_pr_comments(&pr.id).await?; Ok(Json(serde_json::json!({ "comments": comments }))) } + +/// #173 F1 follow-up: `merge_pr` reaches `acquire_write` holding NO admission permit +/// (unlike the push handler, which is capped by the git-push semaphore), so it is one +/// of the callers most likely to meet an exhausted write-lock POOL under load. An +/// exhausted pool is a capacity signal, so the merge must shed 503 + Retry-After +/// (`AppError::Overloaded`) the way the push handler does, not report the generic +/// 500 git error that says nothing about retrying. +#[cfg(test)] +mod lock_pool_shed_tests { + use super::*; + use axum::response::IntoResponse; + use sqlx::PgPool; + + fn seed_repo(owner_did: &str, name: &str) -> crate::db::RepoRecord { + let now = Utc::now(); + crate::db::RepoRecord { + id: Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner_did.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/tmp/{name}"), + forked_from: None, + machine_id: None, + } + } + + /// RED-before/GREEN-after for `merge_pr`. Both directions: the shed while the only + /// lock-pool connection is held by a guard on a DIFFERENT repo (so this is pool + /// capacity, not advisory-lock contention on this repo), and the must-not case + /// once that connection is back. + #[sqlx::test] + async fn merge_pr_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { + let owner = "did:key:zMERGELOCKPOOLOWNERAAAAAAAAAAAAAAAAAAAAA"; + let mut state = crate::test_support::test_state(pool.clone()).await; + // One lock-pool connection with a short checkout timeout, so a single held + // guard exhausts it promptly rather than at the pool default. + state.repo_store = crate::git::repo_store::RepoStore::new( + std::path::PathBuf::from("/tmp/gitlawb-pulls-lockpool"), + None, + crate::git::repo_store::build_lock_pool(&pool, 1, std::time::Duration::from_secs(1)), + ); + + let repo = seed_repo(owner, "lp-merge"); + let repo_id = repo.id.clone(); + state.db.create_repo(&repo).await.expect("seed repo"); + let now = Utc::now().to_rfc3339(); + state + .db + .create_pr(&PullRequest { + id: Uuid::new_v4().to_string(), + repo_id: repo_id.clone(), + number: 1, + title: "lp".to_string(), + body: None, + author_did: owner.to_string(), + source_branch: "feature".to_string(), + target_branch: "main".to_string(), + status: "open".to_string(), + merged_by_did: None, + merged_at: None, + created_at: now.clone(), + updated_at: now, + }) + .await + .expect("seed open PR"); + + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = merge_pr( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-merge".to_string(), 1)), + ) + .await; + let err = shed.expect_err("an exhausted lock pool must fail the call"); + let resp = err.into_response(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "merge_pr: an exhausted write-lock pool must shed 503, not a 500 git error" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .map(|v| v.to_str().unwrap()), + Some("1"), + "merge_pr: a capacity shed must tell the client when to retry" + ); + + // MUST-NOT: with the pool free again the merge is not shed as capacity (it + // fails later on the nonexistent on-disk repo, which is a git 500). + held.release(false).await; + let admitted = merge_pr( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-merge".to_string(), 1)), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, merge_pr must not be shed as capacity; got {:?}", + admitted.err() + ); + } +} diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index c8e9f2a6..49ab1a96 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1244,7 +1244,11 @@ pub(crate) async fn run_post_push_replication_for_test( /// POOL is a capacity signal, not a broken repo, so it sheds 503 + Retry-After the same /// way the admission caps around it do; it used to fall into the generic git 500, which /// tells the client nothing about retrying (#173 F1). Anything else stays a git error. -fn acquire_write_app_error(err: &anyhow::Error, repo: &str) -> AppError { +/// +/// Shared with the non-push `acquire_write` callers (`api/issues.rs`, `api/pulls.rs`) +/// rather than copied: those hold no admission permit, so they meet an exhausted pool +/// first, and a second copy of this mapping would be free to drift from the push path. +pub(crate) fn acquire_write_app_error(err: &anyhow::Error, repo: &str) -> AppError { if err .downcast_ref::() .is_some() From 219fbdbdba13b11751882dbab6be9dd629b87e20 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:44:31 -0500 Subject: [PATCH 23/77] test(node): cover the sweep's boot wiring and prove an interrupted push never reaches the upload Two claims from earlier commits were argued rather than executed. Both are now observed. The legacy-CID sweep's spawn sat inline in main() where no test could reach it, so the batch size, the inter-batch delay, the detachment and the shutdown select were all reasoned. The block moves verbatim into `spawn_legacy_cid_sweep`, which a test calls directly. It seeds rows whose keys are already canonical, so the cost gate skips each and the persisted cursor reads out walk progress cleanly, then asserts the cursor stops at the configured batch boundary, the call returns without blocking, the task is still alive inside its delay, and shutdown ends it. Never spawning, reading the batch or delay from the wrong knob, or removing the shutdown branch each turns it red. What remains reasoned is the single call line inside main(), down from a twenty-six line block. The claim that a disconnect-interrupted push never uploads to Tigris was structural: the tests build the store with no Tigris client, so a counter inside the client arm could never move and would have proved nothing. The counter now sits at the upload decision, before the client is consulted, and the disconnect test asserts zero while the success test asserts exactly one. That positive control is what makes the zero meaningful, and it is itself load-bearing: releasing the success path with failure semantics drops it to zero, and adding a Drop impl that uploads raises the disconnect count to one. The counter observes reaching the upload site, not an S3 request. TigrisClient wraps a concrete SDK client with no injectable endpoint, and plumbing one through would be out of proportion to the risk here. --- crates/gitlawb-node/src/api/repos.rs | 25 +++ crates/gitlawb-node/src/git/repo_store.rs | 38 +++++ crates/gitlawb-node/src/main.rs | 184 +++++++++++++++++----- 3 files changed, 211 insertions(+), 36 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 49ab1a96..89203edb 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -4968,6 +4968,20 @@ mod tests { freed_after_reap, "the write lock must be released once the disconnected push's group is reaped" ); + // The other half of the disconnect invariant, and the reason the guard rides the + // reaper rather than being released there: an interrupted push must not publish a + // half-applied repo. The guard is gone by now (the lock above only frees when it + // is), so the upload site has had its whole chance to be reached. The positive + // control is `receive_pack_success_reclaims_and_releases_the_write_lock`, which + // observes the same counter at 1: without it, a zero here would pass on any build + // where an upload is simply impossible. + assert_eq!( + state.repo_store.tigris_upload_site_reached(), + 0, + "a push interrupted by a client disconnect must not reach the Tigris upload \ + site: publishing a half-applied repo propagates it to every node that later \ + downloads it (#173 F2)" + ); } /// #173 F2, the other half: carrying the write lock through the admission seam must @@ -5056,6 +5070,17 @@ mod tests { write_lock_is_takeable(&probe, key).await, "a completed push must reclaim its write lock and release it synchronously" ); + // POSITIVE CONTROL for the disconnect case's "no upload" assertion. A push that + // completed does reach the Tigris upload site, exactly once, so the zero the + // disconnect test observes is a real difference between the two paths rather than + // an artifact of tests running with no Tigris client configured. Exactly once, + // not at least once: a retried exec race releases with success = false and must + // not count. + assert_eq!( + state.repo_store.tigris_upload_site_reached(), + 1, + "a completed push must reach the Tigris upload site once" + ); } /// #173 F1 (RED-before/GREEN-after): an exhausted repo write-lock POOL is a capacity diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index ebdd7757..02d02efb 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -44,6 +44,21 @@ pub struct RepoStore { /// window and cancel it there. #[cfg(test)] tigris_stall: Option, + /// Test-only counter of how many times a write guard from this store REACHED the + /// Tigris upload site in `release` (the point past the `success` check, where a + /// configured client would be uploaded to). It counts the decision, not a network + /// call: `TigrisClient` takes its endpoint from process-wide AWS env vars and has no + /// injectable seam, so every test runs with `tigris: None` and a counter inside the + /// `Some` arm could never move. Reaching the site is the property under test anyway: + /// an interrupted push must not publish a half-applied repo, and the disconnect path + /// must therefore never get here (#173 F2). + /// + /// Per store rather than a process global, so cases running in parallel do not see + /// each other's uploads, and an `Arc` rather than a `thread_local` because the guard + /// is released from a detached task on another worker thread. Same test-only counter + /// idiom as `ipfs_pin::note_legacy_repair_read`. + #[cfg(test)] + upload_site_reached: Arc, } impl RepoStore { @@ -66,6 +81,14 @@ impl RepoStore { self } + /// Test-only: how many write guards from this store have reached the Tigris upload + /// site. See [`RepoStore::upload_site_reached`]. + #[cfg(test)] + pub fn tigris_upload_site_reached(&self) -> usize { + self.upload_site_reached + .load(std::sync::atomic::Ordering::SeqCst) + } + /// `lock_pool` must come from `build_lock_pool`; a plain pool leaks advisory /// locks on cancellation. pub fn new(repos_dir: PathBuf, tigris: Option, lock_pool: PgPool) -> Self { @@ -76,6 +99,8 @@ impl RepoStore { migrated: Arc::new(Mutex::new(HashSet::new())), #[cfg(test)] tigris_stall: None, + #[cfg(test)] + upload_site_reached: Arc::new(std::sync::atomic::AtomicUsize::new(0)), } } @@ -270,6 +295,8 @@ impl RepoStore { lock_key, lock_conn, tigris: self.tigris.clone(), + #[cfg(test)] + upload_site_reached: Arc::clone(&self.upload_site_reached), }) } @@ -461,6 +488,10 @@ pub struct RepoWriteGuard { /// clears the lock. lock_conn: PoolConnection, tigris: Option, + /// Shared with the store that handed this guard out; see + /// [`RepoStore::upload_site_reached`]. + #[cfg(test)] + upload_site_reached: Arc, } impl RepoWriteGuard { @@ -477,6 +508,13 @@ impl RepoWriteGuard { pub async fn release(mut self, success: bool) { // Upload to Tigris only on success. if success { + // The upload site, recorded for tests before the client is consulted: with + // no injectable seam on `TigrisClient` a counter inside the arm below could + // never move, and it is reaching this point at all that an interrupted push + // must not do (#173 F2). + #[cfg(test)] + self.upload_site_reached + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); if let Some(ref tigris) = self.tigris { if let Err(e) = tigris .upload(&self.owner_slug, &self.repo_name, &self.local_path) diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index d4f4e63d..2a3505a5 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -543,42 +543,7 @@ async fn main() -> Result<()> { }); } - // U4 (#173): one-shot legacy provider-CID repair sweep. Releases before this - // version stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`, - // and this version's `/ipfs/{cid}` resolver withholds any row whose stored key is - // not the raw-content CID. The opportunistic repair on the pin path only fires when - // a push re-carries the object, which normal git negotiation makes it not do, so - // those rows need a walk. DETACHED, never on the boot path: the server below starts - // and serves while this runs, and the sweep's own batch bound plus inter-batch delay - // keep it off the DB's critical path. Its cursor is durable, so a restart mid-walk - // resumes instead of rewinding. - { - let db = state.db.clone(); - let repos_dir = config.repos_dir.clone(); - let git_bin = state.git_bin.clone(); - let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs); - let batch = config.pin_repair_sweep_batch; - let delay = std::time::Duration::from_secs(config.pin_repair_sweep_delay_secs); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - tokio::select! { - stats = ipfs_pin::sweep_legacy_provider_cids( - &repos_dir, &git_bin, git_timeout, batch, delay, &db, - ) => { - if stats.repaired > 0 { - tracing::info!( - scanned = stats.scanned, - repaired = stats.repaired, - "legacy provider-CID sweep finished" - ); - } - } - // Shutdown mid-walk simply drops the run; the persisted cursor means the - // next boot picks up where this one stopped. - _ = shutdown_rx.changed() => {} - } - }); - } + let _legacy_cid_sweep = spawn_legacy_cid_sweep(&state, &config); let router = server::build_router(state.clone()); // Re-register the socket bound at startup — same fd, so there was never a @@ -702,6 +667,49 @@ async fn main() -> Result<()> { Ok(()) } +/// U4 (#173): spawn the one-shot legacy provider-CID repair sweep. Releases before this +/// version stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`, +/// and this version's `/ipfs/{cid}` resolver withholds any row whose stored key is not +/// the raw-content CID. The opportunistic repair on the pin path only fires when a push +/// re-carries the object, which normal git negotiation makes it not do, so those rows +/// need a walk. DETACHED, never on the boot path: the caller keeps serving while this +/// runs, and the sweep's own batch bound plus inter-batch delay keep it off the DB's +/// critical path. Its cursor is durable, so a restart mid-walk resumes instead of +/// rewinding. +/// +/// A named function rather than an inline block in `main` so the WIRING has a seam a +/// test can call: that the task is spawned at all, that it reads its batch and delay +/// from the config knobs rather than some other field, that the caller is not blocked +/// on it, and that the shutdown watcher actually ends it mid-walk. The sweep's own +/// behavior is covered elsewhere; this is the boot-path half. +fn spawn_legacy_cid_sweep(state: &AppState, config: &Config) -> tokio::task::JoinHandle<()> { + let db = state.db.clone(); + let repos_dir = config.repos_dir.clone(); + let git_bin = state.git_bin.clone(); + let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs); + let batch = config.pin_repair_sweep_batch; + let delay = std::time::Duration::from_secs(config.pin_repair_sweep_delay_secs); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + tokio::select! { + stats = ipfs_pin::sweep_legacy_provider_cids( + &repos_dir, &git_bin, git_timeout, batch, delay, &db, + ) => { + if stats.repaired > 0 { + tracing::info!( + scanned = stats.scanned, + repaired = stats.repaired, + "legacy provider-CID sweep finished" + ); + } + } + // Shutdown mid-walk simply drops the run; the persisted cursor means the + // next boot picks up where this one stopped. + _ = shutdown_rx.changed() => {} + } + }) +} + fn spawn_shutdown_signal(tx: watch::Sender) { tokio::spawn(async move { #[cfg(unix)] @@ -1162,6 +1170,110 @@ fn load_or_create_keypair(config: &Config) -> Result { } } +#[cfg(test)] +mod legacy_cid_sweep_wiring_tests { + use super::spawn_legacy_cid_sweep; + use sqlx::PgPool; + use std::time::Duration; + + /// Seed `count` `pinned_cids` rows whose keys are already canonical raw CIDv1, in a + /// known `sha256_hex` order. The sweep's own cost gate skips a raw-CIDv1 row without + /// reading bytes or resolving a repo, so each row is SCANNED (it advances the cursor) + /// and nothing else. That is what makes the cursor a clean readout of how far the + /// walk got, with no dependency on repos on disk. + async fn seed_scannable_rows(pool: &PgPool, count: usize) -> Vec { + let mut shas = Vec::new(); + for i in 1..=count { + let sha = format!("wire{i:02}"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(sha.as_bytes()).to_string(); + assert!( + gitlawb_core::cid::is_raw_cidv1(&cid), + "the seeded key must hit the sweep's raw-CIDv1 skip, not a repair attempt" + ); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind(&sha) + .bind(&cid) + .bind("2020-01-01T00:00:00Z") + .execute(pool) + .await + .unwrap(); + shas.push(sha); + } + shas + } + + /// Poll the persisted sweep cursor until it reaches `want`, or give up. + async fn cursor_reaches(db: &crate::db::Db, want: &str, within: Duration) -> String { + let deadline = std::time::Instant::now() + within; + loop { + let c = db.pin_repair_cursor().await.unwrap(); + if c == want || std::time::Instant::now() >= deadline { + return c; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + + /// #173 U4, the BOOT-PATH half. The sweep's own logic (batching, cursor resumption, + /// terminal vs retryable skips) is covered in `test_support`; what this covers is the + /// wiring `main` performs, which nothing else executes: the task is spawned at all, + /// it takes its batch and delay from the two `pin_repair_sweep_*` knobs rather than + /// some other config field, the caller is not blocked on the walk, and the shutdown + /// watcher ends the run mid-walk. + /// + /// Six scannable rows, batch 2, delay 30s. One pass must land the cursor on exactly + /// the second row and the task must then still be alive in its inter-batch sleep, + /// which pins both knobs at once: a different batch stops at a different row, and a + /// delay that did not come from the knob either finishes the table or leaves the task + /// gone. Shutdown must then end it while four rows are still unwalked. + #[sqlx::test] + async fn the_boot_path_spawns_the_sweep_detached_with_its_configured_knobs(pool: PgPool) { + let state = crate::test_support::test_state(pool.clone()).await; + let shas = seed_scannable_rows(&pool, 6).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + + let mut config = (*state.config).clone(); + config.repos_dir = repos_dir.path().to_path_buf(); + config.pin_repair_sweep_batch = 2; + // Far longer than this test runs, so a task still alive after the first pass can + // only be one that is honoring the configured inter-batch delay. + config.pin_repair_sweep_delay_secs = 30; + + let started = std::time::Instant::now(); + let handle = spawn_legacy_cid_sweep(&state, &config); + let spawn_cost = started.elapsed(); + + let cursor = cursor_reaches(&state.db, &shas[1], Duration::from_secs(10)).await; + assert_eq!( + cursor, shas[1], + "the spawned sweep must run and stop its first pass at the CONFIGURED batch \ + bound (2), leaving the cursor on the second row" + ); + assert!( + spawn_cost < Duration::from_secs(1), + "the sweep must be detached, not awaited on the boot path; the spawn took \ + {spawn_cost:?}" + ); + assert!( + !handle.is_finished(), + "with a 30s inter-batch delay the task must still be sleeping between passes, \ + not finished: a finished task means the delay was not the configured one" + ); + + state.shutdown(); + tokio::time::timeout(Duration::from_secs(10), handle) + .await + .expect("the shutdown watcher must end the sweep, and not after its 30s delay") + .expect("the sweep task must not panic"); + + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + shas[1], + "shutdown must have ended the run MID-walk, with the remaining rows unwalked" + ); + } +} + #[cfg(test)] mod lock_pool_sizing_tests { use super::{lock_pool_size, LOCK_POOL_MAX_CONNECTIONS, LOCK_POOL_PUSH_HEADROOM}; From 3ca35d216dd553e11bc002df39415ba9daa40b7e Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 2 Aug 2026 18:39:33 -0500 Subject: [PATCH 24/77] test(node): restore three guard tests the #174 merge dropped The merge deleted coverage that no longer compiled against the reconciled designs, rather than re-seaming it. Three of those tests have counterparts in the merged tree after all: unlock_error_connection_close_is_bounded and write_guard_dropped_off_runtime_disposes_the_connection came back with close_conn_bounded and the Drop backstop, which the lock-pool design needed after all (the after_release hook cannot free a lock when the unlock errors on a live session). Both now observe the store's derived lock pool rather than the pool handed to for_testing, which is where the guard's connection actually lives. get_by_cid_caps_repos_walked_knob_bounds_the_walks was orphaned when its body was spliced into a sibling test. It drives GITLAWB_IPFS_MAX_REPOS_WALKED, which is the binding cap only now that the gate takes the tighter of the two walk ceilings. The one test not restored is write_guard_release_when_not_locked: the guard has a single construction site and it sits below the lock-or-bail, so a guard that never took the lock cannot exist. --- crates/gitlawb-node/src/api/ipfs.rs | 172 ++++++++++++++++++++++ crates/gitlawb-node/src/git/repo_store.rs | 95 ++++++++++++ 2 files changed, 267 insertions(+) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 02e9add1..04cf75c6 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -3263,6 +3263,178 @@ mod tests { ); } + /// Loop bound (cap N) + F2 truncation verdict: one `/ipfs/{cid}` request against a + /// CID present in many path-scoped repos must not serialize an unbounded number of + /// full-history walks — and cutting a candidate WITHOUT a verdict must not report + /// the object absent. With `ipfs_max_repos_walked = 1` and TWO public, path-scoped + /// repos both carrying the blob, the first candidate is walked (empty allowed-set → + /// a deny VERDICT) and the second is cut by the cap (no verdict), so the fake git's + /// `rev-list` runs exactly once and the request sheds a retryable 503 + Retry-After + /// — never the old false 404 (the blob genuinely sits in the second repo). + /// This drives the GITLAWB_IPFS_MAX_REPOS_WALKED knob specifically. The merge left + /// two walk caps in play, this one and the branch's own history-walk ceiling, and + /// the gate takes the tighter of the two; setting this knob to 1 is what makes it + /// the binding one here. A sibling case covers the ceiling. + /// + /// MUTATION (RED): drop `config.ipfs_max_repos_walked` from the `min()` in the walk + /// gate and both repos are walked (count 2); drop the truncation taint on the skip + /// and the 503 decays to a 404. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_caps_repos_walked_knob_bounds_the_walks(pool: sqlx::PgPool) { + use std::process::Command; + + let tmp = tempfile::TempDir::new().unwrap(); + let walk_log = tmp.path().join("walks.log"); + // Fake git for the WALK: empty refs, `rev-parse` resolves, and each `rev-list` + // appends one line to a log (so the number of walks == the line count) and exits + // with EMPTY output (the allowed-set is empty, so every repo path-gates to a + // `continue` and the request 404s after walking). object_type uses the REAL git, + // so the seeded blob below must genuinely exist. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo walk >> \"{}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + walk_log.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // The bound under test: walk at most one candidate repo per request. + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Seed TWO public repos, each with the SAME blob (same content -> same sha256 OID + // -> same CID) under a path-scoped rule, so both are walk candidates for one CID. + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let mut oid = String::new(); + for (i, name) in ["ipa", "ipb"].iter().enumerate() { + let owner = "z6ipfsN"; + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let work = tmp.path().join(format!("work{i}")); + std::fs::create_dir_all(work.join("src")).unwrap(); + // Identical content in both repos -> identical sha256 blob OID -> one CID. + std::fs::write(work.join("src/secret.txt"), b"loop bound proof\n").unwrap(); + run( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "src/secret.txt"], &work); + run(&["commit", "-q", "-m", "seed"], &work); + run( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp.path(), + ); + if oid.is_empty() { + let out = Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + } + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderBBBBBBBBBBBBBBBBBBBBBBBB".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + // The resolver maps a requested CID back to an oid through the CID index, so a + // bare digest-as-oid CID resolves to nothing and 404s before any repo is + // visited. Register a legacy NULL-provenance row, which is also what routes the + // request to the bounded legacy scan this cap governs. Neither repo serves, so + // the key need not be the content CID. + let cid = seed_legacy_pin(&state, &oid).await; + + let peer: SocketAddr = "203.0.113.90:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = ipfs_router(state).oneshot(req).await.unwrap(); + // The first repo's walk yields the empty allowed-set (deny verdict); the second + // repo NEEDS a walk the cap forbids, so the scan is truncated without a verdict + // on it: retryable 503, never a false 404 for the blob it genuinely carries. + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a walk-cap truncation must shed a retryable 503, not report the object absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 1, + "with the per-request repo-walk cap at 1, only the first candidate repo is \ + walked (the second is cut by the cap), so exactly one walk runs; got {walks}" + ); + } + /// Route rate limit is WIRED (not a silent no-op): the production `build_router` /// attaches an `IpRateLimiter` extension to the `/ipfs/{cid}` route, so a per-IP /// flood is braked with 429. A bare `rate_limit_by_ip` layer with no extension does diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 0b28963d..fb32a0b3 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -2015,6 +2015,101 @@ mod tests { } } + /// F3c (P2): the connection teardown on the failing-unlock path must be BOUNDED. + /// `release` awaits it inline while the global write permit, the per-source permit + /// and the write lease are all still held, and sqlx's `close()` carries no deadline + /// of its own, so a blackholed socket would park every later push to that repo + /// behind three pinned admission resources. + /// + /// What this covers: the deadline itself. A close that never resolves still lets + /// `close_conn_bounded` return, which is the property `release` depends on. What it + /// does NOT cover, and is reasoned rather than run: that sqlx's own `close()` is + /// what stalls in production. Making a real `PgConnection::close` hang needs a + /// blackholed TCP path to Postgres, and the flip has to land after the unlock + /// statement round-trips but before the Terminate write, which is not a seam this + /// module exposes. A never-resolving future is the faithful stand-in for that + /// close, and the F3b tests above already cover that `release` really routes its + /// close through here. + /// + /// Time is paused, so nothing here depends on wall clock: the runtime auto-advances + /// to the next timer, and the assertion is on which timer fired, not on elapsed + /// time. The outer bound is what turns a removed deadline into a failure rather + /// than a hung suite. + /// + /// Load-bearing: drop the `tokio::time::timeout` in `close_conn_bounded` and the + /// inner future never resolves, so the outer bound fires and this fails. + #[tokio::test(start_paused = true)] + async fn unlock_error_connection_close_is_bounded() { + let hanging = std::future::pending::>(); + let outcome = tokio::time::timeout( + UNLOCK_ERROR_CLOSE_TIMEOUT * 4, + close_conn_bounded("boundedclosetest", hanging), + ) + .await; + assert!( + outcome.is_ok(), + "a connection close that never completes must not hold the write lease and \ + both admission permits open-endedly: close_conn_bounded must give up and \ + drop the connection" + ); + } + + /// U8, the off-runtime arm: with no Tokio runtime there is nothing to spawn the + /// unlock onto, and the connection has already been taken out of the guard, so + /// dropping it with no unlock attempted returns it to the pool with the session + /// lock still held. Dropping a `PoolConnection` off a runtime is worse than that: + /// sqlx's return-to-pool path spawns, and its no-runtime fallback panics, so that + /// arm also panics in a destructor. + /// + /// Reached by dropping the guard on a plain `std::thread`, where + /// `Handle::try_current()` fails. + /// + /// Load-bearing: replace the `detach` arm with a plain `drop(conn)` and the join + /// sees sqlx's "requires a Tokio context" panic; `detach` gives up the pool slot, + /// so nothing is spawned and dropping the detached connection closes the socket, + /// which ends the session and frees the lock. + #[sqlx::test] + async fn write_guard_dropped_off_runtime_disposes_the_connection(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); + let owner = "did:key:z6MkDropOffRuntimeProofKKKKKKKKKKKKKKKKKK"; + let name = "dropoffruntimetest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + // The guard's connection lives in the store's DERIVED lock pool, not the pool + // handed to `for_testing`; see `RepoStore::lock_pool`. + let lock_pool = store.lock_pool().clone(); + let size_before = lock_pool.size(); + assert!(size_before > 0, "the lock pool owns the guard's connection"); + + let dropped = std::thread::spawn(move || drop(guard)).join(); + assert!( + dropped.is_ok(), + "dropping a write guard off a Tokio runtime must not panic" + ); + + wait_until( + || lock_pool.size() == size_before - 1, + "the connection of a guard dropped off a runtime to be disposed of rather \ + than returned to the pool with no unlock attempted", + ) + .await; + wait_until_lock_free( + &mut checker, + key, + "a guard dropped off a runtime to end its session so postgres drops the lock", + ) + .await; + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + /// A second pool over the same test database with the idle reaper DISABLED. /// /// `#[sqlx::test]`'s own pool sets `idle_timeout(1s)`, so a connection returned to From 979d3c45bbb152d6ee6ff4edb220bd851c38641f Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 2 Aug 2026 18:39:33 -0500 Subject: [PATCH 25/77] test(node): restore the four u3 drain tests onto the coalescer These covered post-push work being requeued rather than dropped, and were removed in the merge because they drove the replaced requeue loop. The behaviour they check is still there, so they re-seam onto the coalescer without any production change. The seam moved in three ways. The inflight key is the repo identity key, not the row id. A coalescing push now merges its (old, new) tip pairs into the pending slot, so a test that passes an empty vec merges nothing and the drain never laps: every coalesce carries real commit oids. And the full-scan arm is reached through the pending-slot overflow, so the leak test coalesces 1025 pairs and asserts the slot degraded to FullScan before the run. run_encrypt_pin_task_for_test builds the task context the way the production spawn site does and takes a real owner and name, because the drain re-fetches by owner and name; a blank name would resolve Gone on every lap. All four are mutation-proven: drain-does-nothing, stale-empty-rules, full-scan-skips-the-fail-closed-filter, and a leaked inflight key each turn their own test red. --- crates/gitlawb-node/src/api/repos.rs | 48 ++++ crates/gitlawb-node/src/test_support.rs | 362 ++++++++++++++++++++++++ 2 files changed, 410 insertions(+) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index beaf3a01..ae8b752d 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -912,6 +912,54 @@ async fn run_encrypt_pin_task( } } +/// Test-only entry point: build an [`EncryptTaskCtx`] from a test `AppState` (with +/// an overridable `ipfs_api` for a mock Kubo server and an explicit `disk_path` for +/// the fixture repo) and run the real drain task. Keeps `EncryptTaskCtx` and +/// `run_encrypt_pin_task` private to this module. +/// +/// `owner_did` and `repo_name` must name the real seeded row: the drain re-fetches +/// the record by owner/name every lap, so a blank name resolves `Gone` and every +/// lap would pin nothing. +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +pub(crate) async fn run_encrypt_pin_task_for_test( + state: &AppState, + guard: crate::state::EncryptInflightGuard, + disk_path: std::path::PathBuf, + repo_id: String, + owner_did: String, + repo_name: String, + ipfs_api: String, + snapshot_objects: Vec, + snapshot_rules: Option>, + snapshot_is_public: bool, +) { + let ctx = EncryptTaskCtx { + ipfs_api, + repo_path: disk_path, + db: state.db.clone(), + repo_id, + owner_did, + repo_name, + irys_url: String::new(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: state.git_bin.clone(), + git_timeout: std::time::Duration::from_secs(state.config.git_service_timeout_secs), + encrypt_sem: state.git_encrypt_semaphore.clone(), + pin_sem: state.pin_semaphore.clone(), + }; + run_encrypt_pin_task( + ctx, + guard, + snapshot_objects, + snapshot_rules, + snapshot_is_public, + ) + .await; +} + /// Resolve a coalesced-drain iteration's replicable object list. Re-fetches the /// repo record and visibility rules FRESH — rules tightened between the coalesced /// push and its drain must be honored, fail closed: a newly-withheld blob is not diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index ae99a988..3a1edea7 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -10914,4 +10914,366 @@ mod tests { "result includes the deep cert matching the prefix" ); } + + /// Coalesced-drain behavior of the detached post-push encrypt/pin task. + /// + /// A push arriving while a task is in flight does not spawn a second task; its + /// (old_sha, new_sha) tip pairs are merged into the in-flight key's pending slot + /// and the task loop-drains them before releasing the key. These tests drive the + /// real task through `run_encrypt_pin_task_for_test` and assert on the WORK + /// PERFORMED (what is pinned, what is sealed, whether the key is released), not + /// on control flow. The drain re-reads repo state FRESH, so a rule tightened + /// between the coalesced push and its drain must be honored, fail closed. + mod u3_requeue { + use super::*; + use crate::db::VisibilityMode; + use crate::state::{BeginOutcome, PendingWork}; + use std::path::{Path, PathBuf}; + use std::process::Command; + + fn git(args: &[&str], dir: &Path) { + let ok = Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(); + assert!(ok, "git {args:?} failed"); + } + fn oid(rev: &str, dir: &Path) -> String { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(dir) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}: {out:?}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + struct Repo { + _td: tempfile::TempDir, + path: PathBuf, + } + fn init_repo() -> Repo { + let td = tempfile::TempDir::new().unwrap(); + let path = td.path().to_path_buf(); + git(&["init", "-q"], &path); + git(&["config", "user.email", "t@t"], &path); + git(&["config", "user.name", "t"], &path); + Repo { _td: td, path } + } + /// Commit `content` at `rel`, return the blob oid. + fn commit(repo: &Path, rel: &str, content: &str) -> String { + let full = repo.join(rel); + std::fs::create_dir_all(full.parent().unwrap()).unwrap(); + std::fs::write(&full, content).unwrap(); + git(&["add", "."], repo); + git(&["commit", "-qm", rel], repo); + oid(&format!("HEAD:{rel}"), repo) + } + /// Write a loose, UNREACHABLE blob (dangling object). + fn write_dangling_blob(repo: &Path, content: &str) -> String { + let out = Command::new("git") + .args(["hash-object", "-w", "--stdin"]) + .current_dir(repo) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + use std::io::Write; + out.stdin + .as_ref() + .unwrap() + .write_all(content.as_bytes()) + .unwrap(); + let o = out.wait_with_output().unwrap(); + assert!(o.status.success()); + String::from_utf8_lossy(&o.stdout).trim().to_string() + } + fn new_did() -> String { + Keypair::generate().did().to_string() + } + /// Admit push A on the in-flight key, or fail the test. + fn admit(state: &AppState, key: &str) -> crate::state::EncryptInflightGuard { + match state.encrypt_inflight.try_begin(key, Vec::new()) { + BeginOutcome::Admitted(g) => g, + BeginOutcome::Coalesced => panic!("push A must be admitted, nothing is in flight"), + } + } + /// Coalesce push B's tip pairs into the in-flight key, or fail the test. + fn coalesce(state: &AppState, key: &str, pairs: Vec<(String, String)>) { + match state.encrypt_inflight.try_begin(key, pairs) { + BeginOutcome::Coalesced => {} + BeginOutcome::Admitted(_) => { + panic!("push B must coalesce, a task is already in flight") + } + } + } + + /// SCENARIO 2 + 5 (pin half, TAIL-PLACEMENT guard). A coalesced push on a PUBLIC + /// repo with NO path-scoped rule must still drain its pin half: the second + /// push's new object is pinned after the task. RED without the drain (the stale + /// spawn object_list never lists obj2), and RED if the drain sits inside the + /// `has_path_scoped_rule` block (a rules-free repo would never reach it). + #[sqlx::test] + async fn u3_rules_free_public_repo_requeues_pin_half(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-pin"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + // The coalesced push B adds obj2 (present at drain time, NOT in the stale + // push-A spawn object_list). + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Push A admits (guard); push B coalesces its tip pair into the slot. + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Spawn-time (push A) captures are STALE: object_list lists only obj1, no rule. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "push A's object is pinned on the first pass" + ); + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's new object is pinned by the DRAIN lap (RED without \ + the drain, or if the drain sits inside the encrypt gate)" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits clean" + ); + } + + /// SCENARIO 1 + 3 (encrypt half, FRESH re-read). A coalesced push adds a + /// path-scoped rule withholding a blob. The task must re-read rules FRESH on + /// the drain lap and seal the newly-withheld blob's recovery copy. RED without + /// the fresh read (pass one's stale empty rule set seals nothing). + #[sqlx::test] + async fn u3_requeue_seals_blob_withheld_by_coalesced_rule_change(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-enc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let _pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let tip_a = oid("HEAD", &git_repo.path); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + let tip_b = oid("HEAD", &git_repo.path); + + // Coalesced push B changes .gitlawb: withhold /secret/** from anon, grant reader. + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Push A captures are STALE: no rule, public repo. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![], + Some(vec![]), + true, + ) + .await; + + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "the coalesced push's newly-withheld blob is sealed after the DRAIN re-read \ + (RED without the fresh read: pass one's stale empty rules seal nothing)" + ); + assert!(state.encrypt_inflight.is_empty(), "guard key released"); + } + + /// SCENARIO 4 (visibility-leak negative). The drain's full scan must feed + /// `list_all_objects` through the fail-closed filter, never pin it bare: a + /// withheld secret blob and a dangling blob must NOT land in the public pin set. + /// + /// The full scan is forced through the public API: one coalescing push carrying + /// more than the pending tip-pair cap degrades the slot to `PendingWork::FullScan`, + /// which is also the overflow path itself. + #[sqlx::test] + async fn u3_requeue_full_scan_does_not_publicly_pin_withheld_or_dangling(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-leak"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + // Coalesced push adds a new public object and a dangling blob. + let new_pub_oid = commit(&git_repo.path, "public/c.txt", "more public\n"); + let tip = oid("HEAD", &git_repo.path); + let dangling_oid = write_dangling_blob(&git_repo.path, "orphan bytes\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let rules = state.db.list_visibility_rules(&repo.id).await.unwrap(); + + let guard = admit(&state, &key); + // 1025 pairs is one past the pending cap, so the slot degrades to FullScan. + coalesce(&state, &key, vec![(tip.clone(), tip.clone()); 1025]); + assert_eq!( + state.encrypt_inflight.pending_for(&key), + Some(PendingWork::FullScan), + "an overflowing coalesce degrades the pending slot to a forced full scan" + ); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![pub_oid.clone()], + Some(rules), + true, + ) + .await; + + assert!( + state.db.is_pinned(&new_pub_oid).await.unwrap(), + "the coalesced push's new PUBLIC object is pinned by the drain full scan" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "a WITHHELD blob is never publicly pinned by the drain enumeration (leak guard)" + ); + assert!( + !state.db.is_pinned(&dangling_oid).await.unwrap(), + "a DANGLING blob is never publicly pinned by the drain enumeration (leak guard)" + ); + // The withheld blob still gets its ENCRYPTED recovery copy (not a public pin). + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "withheld blob is sealed as an encrypted recovery copy, not pinned in the clear" + ); + } + + /// SCENARIO 8 (no-coalesce happy path). A single push with no coalesced follower + /// runs exactly one pass, pins its object, and releases the key. No drain lap. + #[sqlx::test] + async fn u3_no_coalesce_single_pass_pins_and_releases(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-happy"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // No second try_begin: nothing is ever merged into the pending slot. + let guard = admit(&state, &key); + assert_eq!( + state.encrypt_inflight.pending_for(&key), + Some(PendingWork::Tips(vec![])), + "clean, no coalesce" + ); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "the single push's object is pinned" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the key is released after one pass" + ); + } + } } From 8557a12e59664834e693091dd433436d40d8ad89 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 2 Aug 2026 18:50:03 -0500 Subject: [PATCH 26/77] test(node): add a test-only fault seam for the drain's two re-reads A real Postgres pool will not fail on demand, so the drain's error arms have no way to be driven from a test. This ports the injection table the removed requeue loop used, keyed on the task's repo id, which is a fresh uuid per test so parallel cases cannot cross-inject. The two reads now go through drain_get_repo and drain_list_rules, whose bodies consult the seam under cfg(test) and then make the same call as before. Error handling is untouched: the repo read keeps its arms, the rules read keeps its ok() collapse, and no retry exists yet. The suite is unchanged at 856, which is the point. inject and counters carry a dead_code allow until the tests that call them land. --- crates/gitlawb-node/src/api/repos.rs | 102 ++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index ae8b752d..e3303855 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -960,6 +960,104 @@ pub(crate) async fn run_encrypt_pin_task_for_test( .await; } +/// Test-only fault-injection seam for the drain re-reads in +/// `resolve_drain_object_list`. The behavior worth testing lives on the `Err` arm of +/// the two re-reads, which a real Postgres pool will not produce on demand, so the two +/// reads go through the wrappers below and consult this table first. Keyed by `repo_id` +/// (a fresh uuid per test) so tests running in parallel in one process cannot see each +/// other's injections, and it also records the ATTEMPT counts the retry-bound +/// assertions key on. +#[cfg(test)] +pub(crate) mod drain_faults { + use std::collections::HashMap; + use std::sync::{Mutex, OnceLock}; + + #[derive(Default, Clone, Copy, Debug)] + pub(crate) struct Counters { + pub(crate) repo_read_failures_left: usize, + pub(crate) rules_read_failures_left: usize, + pub(crate) repo_read_attempts: usize, + pub(crate) rules_read_attempts: usize, + } + + fn table() -> &'static Mutex> { + static TABLE: OnceLock>> = OnceLock::new(); + TABLE.get_or_init(|| Mutex::new(HashMap::new())) + } + + /// Make the next `repo_read_failures` repo re-reads and the next + /// `rules_read_failures` rule re-reads for `repo_id` return `Err`, then succeed. + /// Only the tests that drive the retry arms call it, so it is dead until they land. + #[allow(dead_code)] + pub(crate) fn inject(repo_id: &str, repo_read_failures: usize, rules_read_failures: usize) { + table().lock().unwrap().insert( + repo_id.to_string(), + Counters { + repo_read_failures_left: repo_read_failures, + rules_read_failures_left: rules_read_failures, + ..Default::default() + }, + ); + } + + /// Observed attempt counts (and remaining injections) for `repo_id`. + /// Only the tests that assert the retry bound call it, so it is dead until they land. + #[allow(dead_code)] + pub(crate) fn counters(repo_id: &str) -> Counters { + table() + .lock() + .unwrap() + .get(repo_id) + .copied() + .unwrap_or_default() + } + + /// Production-path hook: count one repo re-read attempt, return whether it must fail. + pub(crate) fn take_repo_read(repo_id: &str) -> bool { + let mut map = table().lock().unwrap(); + let c = map.entry(repo_id.to_string()).or_default(); + c.repo_read_attempts += 1; + if c.repo_read_failures_left > 0 { + c.repo_read_failures_left -= 1; + return true; + } + false + } + + /// Production-path hook: count one rules re-read attempt, return whether it must fail. + pub(crate) fn take_rules_read(repo_id: &str) -> bool { + let mut map = table().lock().unwrap(); + let c = map.entry(repo_id.to_string()).or_default(); + c.rules_read_attempts += 1; + if c.rules_read_failures_left > 0 { + c.rules_read_failures_left -= 1; + return true; + } + false + } +} + +/// The drain's repo re-read, behind the test-only fault seam above. +async fn drain_get_repo(ctx: &EncryptTaskCtx) -> anyhow::Result> { + #[cfg(test)] + if drain_faults::take_repo_read(&ctx.repo_id) { + return Err(anyhow::anyhow!("injected repo re-read failure")); + } + ctx.db.get_repo(&ctx.owner_did, &ctx.repo_name).await +} + +/// The drain's visibility-rule re-read, behind the test-only fault seam above. +async fn drain_list_rules( + ctx: &EncryptTaskCtx, + record_id: &str, +) -> anyhow::Result> { + #[cfg(test)] + if drain_faults::take_rules_read(&ctx.repo_id) { + return Err(anyhow::anyhow!("injected visibility-rule re-read failure")); + } + ctx.db.list_visibility_rules(record_id).await +} + /// Resolve a coalesced-drain iteration's replicable object list. Re-fetches the /// repo record and visibility rules FRESH — rules tightened between the coalesced /// push and its drain must be honored, fail closed: a newly-withheld blob is not @@ -980,7 +1078,7 @@ async fn resolve_drain_object_list( Option>, bool, )> { - let record = match ctx.db.get_repo(&ctx.owner_did, &ctx.repo_name).await { + let record = match drain_get_repo(ctx).await { Ok(Some(r)) => r, Ok(None) => { tracing::warn!( @@ -1002,7 +1100,7 @@ async fn resolve_drain_object_list( // fresh by owner/name, and a delete+re-create between spawn and drain gives // the row a NEW id — rules read against the stale id come back empty and // would fail open for the new row. - let rules_opt = ctx.db.list_visibility_rules(&record.id).await.ok(); + let rules_opt = drain_list_rules(ctx, &record.id).await.ok(); let (_announce, withheld) = replication_withheld_set( ctx.encrypt_sem.clone(), rules_opt.clone(), From d1fc9bc835cd722da9751ea498289d7b941eb15d Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 2 Aug 2026 20:16:58 -0500 Subject: [PATCH 27/77] fix(node): retry the coalesced drain's re-read instead of losing the push The drain read the repo row and its visibility rules once each. Either read failing dropped that coalesced push's pins and recovery copy for good: the repo read returned None on error, and the rules read collapsed through ok(), which makes the repo read as not announceable. Both fail closed, so nothing leaked, but nothing sweeps the lost work up either. A single database blip was enough. resolve_drain_object_list now refreshes through a bounded retry: three attempts, 50ms doubling, and either read failing is transient. A missing repo row is the one terminal answer, so it releases at once and spends no retry budget. Exhaustion logs at ERROR with the repo id and attempt count, so giving up is observable rather than silent. The coalescing loop is untouched. It already has the shape the retry needs: a refresh that gives up skips the pin call, and the next finish_or_take_pending picks up anything that coalesced while the retry was running. Eight restored tests cover it, written and observed red against the seam-only tree first. The five the plan predicted red came up red for the reasons it predicted, and the three controls stayed green. All eight are mutation-proven: no-retry, an off-by-one bound, gone burning budget, a rules error read as empty, an unfiltered candidate set, a duplicated refresh, break-on-giveup, and a pending slot that never drains each turn their own test red. --- crates/gitlawb-node/src/api/repos.rs | 100 +++- crates/gitlawb-node/src/test_support.rs | 673 ++++++++++++++++++++++++ 2 files changed, 759 insertions(+), 14 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index e3303855..fa812069 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -987,8 +987,6 @@ pub(crate) mod drain_faults { /// Make the next `repo_read_failures` repo re-reads and the next /// `rules_read_failures` rule re-reads for `repo_id` return `Err`, then succeed. - /// Only the tests that drive the retry arms call it, so it is dead until they land. - #[allow(dead_code)] pub(crate) fn inject(repo_id: &str, repo_read_failures: usize, rules_read_failures: usize) { table().lock().unwrap().insert( repo_id.to_string(), @@ -1001,8 +999,6 @@ pub(crate) mod drain_faults { } /// Observed attempt counts (and remaining injections) for `repo_id`. - /// Only the tests that assert the retry bound call it, so it is dead until they land. - #[allow(dead_code)] pub(crate) fn counters(repo_id: &str) -> Counters { table() .lock() @@ -1058,6 +1054,84 @@ async fn drain_list_rules( ctx.db.list_visibility_rules(record_id).await } +/// Attempts allowed for the drain re-read before the lap gives up. The coalesced +/// push's work is already out of the pending slot (`finish_or_take_pending` took it +/// in the same critical section that kept the key), and there is no reconciliation +/// sweep to re-derive it: a transient read error must be RETRIED here or that push's +/// pin/encrypt pass is gone. The bound keeps a sustained outage from spinning +/// forever; on exhaustion the work is still lost (the pre-existing residual), but +/// the give-up is logged at ERROR so it is observable instead of silent. +const DRAIN_REREAD_MAX_ATTEMPTS: usize = 3; + +/// Backoff before the next re-read attempt. Doubles per attempt. +const DRAIN_REREAD_BACKOFF: std::time::Duration = std::time::Duration::from_millis(50); + +/// The outcome of the drain's fresh state re-read, keeping the three cases distinct +/// that a single `Err => None` collapses into one: a usable refresh, a repo that +/// genuinely no longer exists (terminal, and NOT a retry), and a transient read +/// failure (retryable). +enum DrainRefresh { + State { + /// Boxed only to keep the enum small: `RepoRecord` dwarfs the other two + /// variants, which carry nothing (clippy::large_enum_variant). + record: Box, + rules: Vec, + }, + Gone, + Failed, +} + +/// Re-read repo state for a drain lap, retrying transient read errors. +/// +/// Both reads are retryable and neither may be read as an absence: an `Err` from the +/// repo row is not "the repo is gone", and an `Err` from the rule list is not "this +/// repo has no rules" (`.ok()` made those indistinguishable, and a `None` rule set +/// makes `replication_withheld_set` return `None`, which skips the entire lap). Only +/// `Ok(None)` on the repo row is a terminal absence, and it consumes no retry budget. +/// +/// The whole `RepoRecord` comes back, not just its rules and flags: the caller writes +/// against `record.id` from this FRESH re-fetch, never `ctx.repo_id` frozen at spawn. +async fn drain_refresh_state(ctx: &EncryptTaskCtx) -> DrainRefresh { + let mut backoff = DRAIN_REREAD_BACKOFF; + for attempt in 1..=DRAIN_REREAD_MAX_ATTEMPTS { + let record = match drain_get_repo(ctx).await { + Ok(Some(rec)) => Box::new(rec), + Ok(None) => return DrainRefresh::Gone, + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, err = %e, attempt, + "coalesced drain: repo re-read failed; retrying" + ); + tokio::time::sleep(backoff).await; + backoff *= 2; + continue; + } + }; + // record.id, never the spawn-time ctx.repo_id: the record above is re-fetched + // fresh by owner/name, and a delete+re-create between spawn and drain gives + // the row a NEW id - rules read against the stale id come back empty and + // would fail open for the new row. + match drain_list_rules(ctx, &record.id).await { + Ok(rules) => return DrainRefresh::State { record, rules }, + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, err = %e, attempt, + "coalesced drain: visibility-rule re-read failed; retrying" + ); + tokio::time::sleep(backoff).await; + backoff *= 2; + } + } + } + tracing::error!( + repo = %ctx.repo_id, + attempts = DRAIN_REREAD_MAX_ATTEMPTS, + "coalesced drain: re-read failed on every attempt; the coalesced push's \ + pin/encrypt pass is dropped (no reconciliation sweep re-derives it)" + ); + DrainRefresh::Failed +} + /// Resolve a coalesced-drain iteration's replicable object list. Re-fetches the /// repo record and visibility rules FRESH — rules tightened between the coalesced /// push and its drain must be honored, fail closed: a newly-withheld blob is not @@ -1078,29 +1152,27 @@ async fn resolve_drain_object_list( Option>, bool, )> { - let record = match drain_get_repo(ctx).await { - Ok(Some(r)) => r, - Ok(None) => { + // Both re-reads are bounded-retried: a transient blip must not discard the + // coalesced push's work (`finish_or_take_pending` already took it out of the + // pending slot and no sweep re-derives it). The rules come back from the same + // refresh, read against the FRESH record.id, never the spawn-time ctx.repo_id. + let (record, rules_opt) = match drain_refresh_state(ctx).await { + DrainRefresh::State { record, rules } => (*record, Some(rules)), + DrainRefresh::Gone => { tracing::warn!( repo = %ctx.repo_id, "coalesced drain: repo record is gone; dropping the pending work" ); return None; } - Err(e) => { + DrainRefresh::Failed => { tracing::warn!( repo = %ctx.repo_id, - err = %e, "coalesced drain: repo re-fetch failed; pinning nothing (fail closed)" ); return None; } }; - // record.id, never the spawn-time ctx.repo_id: the record above is re-fetched - // fresh by owner/name, and a delete+re-create between spawn and drain gives - // the row a NEW id — rules read against the stale id come back empty and - // would fail open for the new row. - let rules_opt = drain_list_rules(ctx, &record.id).await.ok(); let (_announce, withheld) = replication_withheld_set( ctx.encrypt_sem.clone(), rules_opt.clone(), diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 3a1edea7..7da4e269 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -11275,5 +11275,678 @@ mod tests { "the key is released after one pass" ); } + + mod u2_reread_retry { + use super::*; + use crate::api::repos::drain_faults; + + /// Process-wide tracing capture so a test can assert the give-up is logged at + /// ERROR. A global default subscriber can only be installed once per process, + /// so it is shared by every test here and assertions filter on the repo id, + /// which is a fresh uuid per test. + mod logcap { + use std::sync::{Arc, Mutex, OnceLock}; + use tracing::{Event, Level, Subscriber}; + use tracing_subscriber::layer::{Context, Layer}; + use tracing_subscriber::prelude::*; + + type Lines = Arc>>; + + fn lines() -> &'static Lines { + static LINES: OnceLock = OnceLock::new(); + LINES.get_or_init(|| Arc::new(Mutex::new(Vec::new()))) + } + + struct Capture; + impl Layer for Capture { + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + struct V(String); + impl tracing::field::Visit for V { + fn record_debug( + &mut self, + field: &tracing::field::Field, + value: &dyn std::fmt::Debug, + ) { + self.0.push_str(&format!(" {}={:?}", field.name(), value)); + } + } + let mut v = V(String::new()); + event.record(&mut v); + lines() + .lock() + .unwrap() + .push((*event.metadata().level(), v.0)); + } + } + + pub(super) fn install() { + static ONCE: OnceLock<()> = OnceLock::new(); + ONCE.get_or_init(|| { + let _ = tracing::subscriber::set_global_default( + tracing_subscriber::registry().with(Capture), + ); + }); + } + + pub(super) fn errors_containing(needle: &str) -> Vec { + lines() + .lock() + .unwrap() + .iter() + .filter(|(lvl, msg)| *lvl == Level::ERROR && msg.contains(needle)) + .map(|(_, msg)| msg.clone()) + .collect() + } + } + + /// SCENARIO 1. The repo re-read fails once, then succeeds: the drain lap + /// must still RUN, under the refreshed state, and pin the coalesced push's + /// object. RED before the fix (the single `Err` returned `None`, the lap + /// pinned nothing, and `finish_or_take_pending` had already taken the + /// pending work out of the slot, so it was gone for good). + #[sqlx::test] + async fn u2_transient_repo_reread_failure_is_retried_and_work_lands(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-retry"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + // The coalesced push B adds obj2, absent from push A's spawn captures. + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // One transient repo re-read failure, then the real DB answers. + drain_faults::inject(&repo.id, 1, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's object is pinned after the retried re-read (RED \ + before this unit: the Err arm dropped the lap and the work with it)" + ); + let c = drain_faults::counters(&repo.id); + assert_eq!( + c.repo_read_attempts, 2, + "the failed re-read is retried exactly once before it succeeds" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits" + ); + } + + /// SCENARIO 2. Every re-read attempt fails: the loop must give up on a BOUND + /// (asserted as a literal, so raising or removing the bound goes RED) and log + /// the give-up at ERROR so the residual loss is observable rather than silent. + #[sqlx::test] + async fn u2_sustained_repo_reread_failure_is_bounded_and_logged(pool: PgPool) { + logcap::install(); + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-bounded"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Far more failures than the bound allows: the outage never clears. + drain_faults::inject(&repo.id, 10_000, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + let c = drain_faults::counters(&repo.id); + assert_eq!( + c.repo_read_attempts, 3, + "the re-read is bounded at 3 attempts; unbounded retry or a raised \ + bound must fail here" + ); + assert!( + !state.db.is_pinned(&obj2).await.unwrap(), + "with the read never succeeding there is nothing fresh to act on" + ); + let errs = logcap::errors_containing(&repo.id); + assert!( + !errs.is_empty(), + "the exhausted drain re-read is logged at ERROR with the repo id, so \ + the residual work loss is observable; captured: {errs:?}" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is still released on the give-up path" + ); + } + + /// SCENARIO 3. `Ok(None)` (the repo was deleted during the in-flight window) + /// is NOT a transient failure: it must release immediately without burning the + /// retry budget. The repo row is never created, so the re-read legitimately + /// returns `Ok(None)`. + #[sqlx::test] + async fn u2_repo_gone_releases_without_consuming_retries(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let missing_id = uuid::Uuid::new_v4().to_string(); + let missing_name = "u2-gone".to_string(); + let key = crate::state::repo_identity_key(&owner, &missing_name); + let git_repo = init_repo(); + let _obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let _obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let server = mockito::Server::new_async().await; + + drain_faults::inject(&missing_id, 0, 0); + + let guard = admit(&state, &key); + // A real pair, so a drain lap actually runs and reaches the re-read. + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Empty object list: pass one touches no pin rows for a repo that is gone. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + missing_id.clone(), + owner.clone(), + missing_name.clone(), + server.url(), + vec![], + Some(vec![]), + true, + ) + .await; + + let c = drain_faults::counters(&missing_id); + assert_eq!( + c.repo_read_attempts, 1, + "a deleted repo is a terminal answer, never retried" + ); + assert_eq!( + c.rules_read_attempts, 0, + "no rules read is attempted once the repo row is gone" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released cleanly" + ); + } + + /// SCENARIO 4. A failed visibility-rule read is transient, never an empty + /// policy. RED before the fix, where `.ok()` made "the rules read failed" and + /// "this repo has no rules" the same value: the withheld blob was then neither + /// sealed nor covered, because a `None` rule set skips the entire lap. + #[sqlx::test] + async fn u2_transient_rules_read_failure_is_retried_not_read_as_empty(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u2-rules"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let tip_a = oid("HEAD", &git_repo.path); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + let tip_b = oid("HEAD", &git_repo.path); + + // The coalesced push B is what added the path-scoped rule. + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + &[reader], + &owner, + ) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // The repo row reads fine; the RULES read is the one that blips. + drain_faults::inject(&repo.id, 0, 1); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Push A's captures are stale: no rule, nothing withheld. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![pub_oid.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "the withheld blob is sealed under the RETRIED rule set (RED with \ + list_visibility_rules(..).ok(): an empty policy seals nothing)" + ); + let c = drain_faults::counters(&repo.id); + assert_eq!( + c.rules_read_attempts, 2, + "the failed rules read is retried, not collapsed into an empty rule set" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "the withheld blob is never pinned in the clear by the drain" + ); + } + + /// SCENARIO 5. The fault-free control for scenario 4: the rules applied by the + /// drain are the COALESCED push's fresh ones, never the spawn-time capture, + /// and the retry path does not perturb that (exactly one read of each). + #[sqlx::test] + async fn u2_requeue_applies_fresh_rules_not_spawn_captures(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u2-fresh"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let tip_a = oid("HEAD", &git_repo.path); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + let tip_b = oid("HEAD", &git_repo.path); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + &[reader], + &owner, + ) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + drain_faults::inject(&repo.id, 0, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![pub_oid.clone()], + Some(vec![]), + true, + ) + .await; + + let c = drain_faults::counters(&repo.id); + assert_eq!( + (c.repo_read_attempts, c.rules_read_attempts), + (1, 1), + "a healthy DB is read exactly once per drain lap" + ); + assert!( + state.db.is_pinned(&pub_oid).await.unwrap(), + "the visible object is pinned under the fresh rules" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "the freshly-read rule withholds the secret blob (the spawn-time \ + capture had no rules at all)" + ); + } + + /// SCENARIO 6. Regression guard on the property the fix must not disturb: the + /// finish-or-take critical section is atomic, so a push coalescing during it is + /// still covered by exactly one more lap, and the key is released after. + #[sqlx::test] + async fn u2_coalesced_push_still_covered_by_exactly_one_requeue_pass(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-coalesce"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + drain_faults::inject(&repo.id, 0, 0); + + let guard = admit(&state, &key); + // Push B lands during the in-flight window: its tip pair is merged. + coalesce(&state, &key, vec![(tip_a.clone(), tip_b.clone())]); + assert_eq!( + state.encrypt_inflight.pending_for(&key), + Some(PendingWork::Tips(vec![(tip_a, tip_b)])), + "the coalesced push recorded its work in the pending slot" + ); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert_eq!( + drain_faults::counters(&repo.id).repo_read_attempts, + 1, + "one coalesced push means exactly one drain lap, no re-spin" + ); + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "push A's object is pinned" + ); + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's object is covered by the drain lap" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the key is released once the task is clean" + ); + } + + /// Wait for `finish_or_take_pending` to take the pending work out of the slot + /// (`Tips(nonempty)` -> `Tips(empty)`), which is the exact instant the task + /// enters `drain_refresh_state`'s retry window. Deterministic, so the + /// coalescing push below lands INSIDE that window rather than on a sleep + /// guess. `None` means the key is already gone (the task exited), which the + /// caller reports as its own failure. + async fn wait_for_drain_window( + inflight: &crate::state::EncryptInflight, + key: &str, + ) -> bool { + for _ in 0..5_000 { + match inflight.pending_for(key) { + Some(PendingWork::Tips(acc)) if acc.is_empty() => return true, + None => return false, + Some(_) => {} + } + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + false + } + + /// SCENARIO 7 (RED-before/GREEN-after). A push that coalesces WHILE the + /// re-read is retrying must not be thrown away when that re-read finally + /// gives up. Breaking the drain loop on the give-up would let + /// `EncryptInflightGuard::drop` remove the key with push C's work still + /// recorded, and push C's lap would never run: a silent drop with no + /// reconciliation sweep behind it. + /// + /// Exactly `DRAIN_REREAD_MAX_ATTEMPTS` injected repo-read faults, so the + /// first refresh exhausts its budget and the DB is healthy for the next one. + /// Push C coalesces inside that window. + #[sqlx::test] + async fn u2_failed_reread_keeps_a_push_that_coalesced_during_the_window(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-window"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj_a = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let _obj_b = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + let obj_c = commit(&git_repo.path, "c.txt", "three\n"); + let tip_c = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Exactly the bound: the FIRST refresh burns all three attempts and gives + // up; every later refresh sees a healthy DB. + drain_faults::inject(&repo.id, 3, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a.clone(), tip_b.clone())]); + + // Push C lands during the retry window, after the loop already took push + // B's pending work out of the slot. It MUST carry a real tip pair: an + // empty merge leaves the slot empty and no extra lap runs at all. + let inflight = state.encrypt_inflight.clone(); + let watch_key = key.clone(); + let coalesced = tokio::spawn(async move { + if !wait_for_drain_window(&inflight, &watch_key).await { + return false; + } + matches!( + inflight.try_begin(&watch_key, vec![(tip_b, tip_c)]), + BeginOutcome::Coalesced + ) + }); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj_a.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + coalesced.await.expect("coalescing task"), + "push C must have coalesced inside the retry window for this test to \ + mean anything" + ); + assert!( + state.db.is_pinned(&obj_c).await.unwrap(), + "the push that coalesced during the retry window must still get a lap \ + once the DB recovers (RED if the give-up breaks the loop: the pending \ + work was already taken, so the lap was dropped with nothing to \ + re-derive it)" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits" + ); + } + + /// SCENARIO 8 (the sustained-outage guard on the fall-through). Continuing the + /// loop on a give-up means `finish_or_take_pending` runs again, so a DB that + /// never recovers must still TERMINATE rather than spin. It does: an extra lap + /// only happens when a push actually coalesced, and each lap pays a full + /// bounded re-read (3 attempts with backoff). One coalescing push during the + /// window buys exactly one extra lap: 6 repo-read attempts, then exit. + #[sqlx::test] + async fn u2_sustained_failure_with_a_coalesce_terminates_after_one_more_lap( + pool: PgPool, + ) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-sustained-window"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj_a = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let _obj_b = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + let _obj_c = commit(&git_repo.path, "c.txt", "three\n"); + let tip_c = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // The outage never clears. + drain_faults::inject(&repo.id, 10_000, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a.clone(), tip_b.clone())]); + + let inflight = state.encrypt_inflight.clone(); + let watch_key = key.clone(); + let coalesced = tokio::spawn(async move { + if !wait_for_drain_window(&inflight, &watch_key).await { + return false; + } + matches!( + inflight.try_begin(&watch_key, vec![(tip_b, tip_c)]), + BeginOutcome::Coalesced + ) + }); + + // The watchdog is the real assertion: a loop that re-spins without the + // pending gate would never return here. + tokio::time::timeout( + std::time::Duration::from_secs(60), + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj_a.clone()], + Some(vec![]), + true, + ), + ) + .await + .expect( + "the task must terminate under a sustained outage; a fall-through that \ + does not gate on the pending slot spins forever", + ); + + assert!( + coalesced.await.expect("coalescing task"), + "push C must have coalesced inside the retry window" + ); + assert_eq!( + drain_faults::counters(&repo.id).repo_read_attempts, + 6, + "one coalescing push buys exactly one more bounded re-read lap \ + (3 + 3 attempts), never an unbounded retry" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released on the give-up path" + ); + } + } } } From aeb6895141637f65972bde097ea5232eafe2b45d Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 2 Aug 2026 20:16:58 -0500 Subject: [PATCH 28/77] test(node): stop the repos.rs structural gates splitting at the wrong marker Three gates scanned the "production half" of api/repos.rs by splitting at the first cfg(test) attribute. Adding test-only items above the code they check moved that split point above every line being scanned, so all three failed looking for production code that was still right there. They now split at the test module, which is what they meant. This is the same defect already fixed in the ipfs gate during the merge, and no split-on-the- attribute site remains. Both behavior-bearing gates were re-proven load-bearing: F2 goes red when the Pinata object set is re-derived before the pin permit is taken, and the tail gate goes red when release stops consuming the same success flag. F3 is a presence check whose marker cannot be removed by a single compiling edit, so it is not mutation-proven here. --- crates/gitlawb-node/tests/inv22_gates.rs | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 7fb25512..33815022 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -147,8 +147,12 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { // recovery copies are absent until an unrelated later push. Scan only the // production half of the file — the u5 tests in its `mod tests` also name the // drain call, and matching them would make this check vacuous. + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. let repos_production = repos - .split("#[cfg(test)]") + .split("\nmod tests {") .next() .expect("split always yields a first chunk"); assert!( @@ -326,8 +330,12 @@ fn f6_ipfs_metadata_queries_are_deadline_wrapped() { #[test] fn f2_pinata_enqueues_refs_not_retained_object_lists() { let repos = src("api/repos.rs"); + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. let production = repos - .split("#[cfg(test)]") + .split("\nmod tests {") .next() .expect("split always yields a first chunk"); @@ -386,8 +394,12 @@ fn f2_pinata_enqueues_refs_not_retained_object_lists() { fn f3_second_writer_leased_until_reap() { let repos = src("api/repos.rs"); let smart_http = src("git/smart_http.rs"); + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. let repos_production = repos - .split("#[cfg(test)]") + .split("\nmod tests {") .next() .expect("split always yields a first chunk"); @@ -511,8 +523,12 @@ fn inv22_ipfs_walk_admission_reaches_every_blocking_site() { fn inv22_replication_tail_spawns_at_the_durability_boundary() { let repos = src("api/repos.rs"); // Production half only — the tests below name these identifiers too. + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. let production = repos - .split("#[cfg(test)]") + .split("\nmod tests {") .next() .expect("split always yields a first chunk"); From 9bba8f99343549b30896a3ebe21e10b93462b273 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 3 Aug 2026 01:16:52 -0500 Subject: [PATCH 29/77] test(node): restore the ipfs-max-repos-walked range test the #174 merge dropped The merge kept the knob and its clap range but carried over only the default-value assertion, so nothing asserted that 0 and 1048577 are rejected. Restored verbatim from the #174 side, beside its ipfs_max_repo_visits sibling. Load-bearing: widening the range to 0..=2_097_152 turns it red. --- crates/gitlawb-node/src/config.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 38c92197..e88611d6 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -1025,6 +1025,24 @@ mod tests { }); } + #[test] + fn ipfs_max_repos_walked_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_max_repos_walked, + 64 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "8"]) + .ipfs_max_repos_walked, + 8 + ); + // 0 would walk no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1048577"]).is_err() + ); + } + #[test] fn ipfs_max_repo_visits_defaults_and_rejects_out_of_range() { assert_eq!( From 8f3a52175aa0608cb1fe729837e3664214824198 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 10 Aug 2026 01:24:46 -0500 Subject: [PATCH 30/77] docs(node): correct the walk-cap and db-pool operator docs Two doc drifts jatmn flagged on #173, both docs-only. The /ipfs walk cap is min(internal ceiling, GITLAWB_IPFS_MAX_REPOS_WALKED), where the internal ceiling is MAX_PIN_SOURCES + 1 = 17. config.rs, README, and .env.example all described a max() floor instead, so an operator reading them would think the default of 64 was the effective ceiling and that raising the knob bought headroom. Neither is true. The runtime is not changed: two tests set the knob to 1 and depend on the cap actually being 1, so wiring the documented floor would make the knob inert and break them. db_max_connections has defaulted to 48 for a while and four comments still said 20. Two of them did arithmetic on it. main.rs and repo_store.rs justified the separate advisory-lock pool by "20 is below 32", which is now false and made the separation look like a consequence of the defaults; it is structural, since a push holds its connection for the whole receive-pack at any pool size. Also adds a RUN-A-NODE troubleshooting entry for the boot failure an in-place upgrade hits when it is still configured with the old pool size of 20. --- .env.example | 4 +++- README.md | 2 +- crates/gitlawb-node/src/config.rs | 17 +++++++++++------ crates/gitlawb-node/src/git/repo_store.rs | 11 +++++++---- crates/gitlawb-node/src/main.rs | 15 +++++++++------ docs/RUN-A-NODE.md | 2 ++ 6 files changed, 33 insertions(+), 18 deletions(-) diff --git a/.env.example b/.env.example index 4e01e008..174319a3 100644 --- a/.env.example +++ b/.env.example @@ -220,7 +220,9 @@ GITLAWB_IPFS_MAX_LEGACY_PROBES=256 # Max EXPENSIVE path-scope visibility walks per single /ipfs request (only a # blob in a path-scoped repo costs a full-history walk). Over-cap repos are # skipped without a verdict and the scan continues; if the object is then found -# nowhere the request sheds a retryable 503 instead of a false 404. Default 64. +# nowhere the request sheds a retryable 503 instead of a false 404. The effective +# cap is the tighter of this value and the node's internal per-request ceiling of +# 17, so values above 17 have no effect; lower values do tighten it. Default 64. GITLAWB_IPFS_MAX_REPOS_WALKED=64 # Ceiling on repos one /ipfs request may VISIT past the visibility gate. Each # visit costs a repo acquire — on a Tigris cache miss a full archive download, diff --git a/README.md b/README.md index c3c99482..3b2e1047 100644 --- a/README.md +++ b/README.md @@ -353,7 +353,7 @@ Important node settings: | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | | `GITLAWB_IPFS_MAX_LEGACY_PROBES` | Max legacy (NULL-provenance) repos probed per `/ipfs/{cid}` request, bounding the scan-fallback fan-out. A truncated scan returns a retryable 503, not a false 404. Default 256. | -| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Raised to `MAX_PIN_SOURCES + 1` if set below it, so a provenanced request is never truncated before its full source set is tried. Default 64. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. The effective cap is the tighter of this knob and the node's internal per-request history-walk ceiling of 17 (`MAX_PIN_SOURCES + 1`), so a value above 17 has no effect while a value below it does tighten the cap. Default 64. | | `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate. Also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | | `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting and each also run under their own deadline (the lesser of `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` and the remaining budget), reaped via process-group teardown, so a hung `cat-file` cannot hold the request's walk slot past it. One hang path is still unbounded: the probe's object-store readability check is a plain filesystem sweep with nothing to reap, so a wedged filesystem can hold the slot past the deadline. Default 600. Accepted range is 1 to 3153600000 (100 years), since the node derives a deadline from this value and a larger one cannot be represented. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index e88611d6..30fae910 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -345,8 +345,8 @@ pub struct Config { /// CONNECTION BUDGET. A push holds a Postgres connection from the node's separate /// advisory-lock pool for the whole receive-pack, and that pool is sized from this /// knob (this value + 8, clamped to 64 in `main.rs`). The node's total ceiling is - /// therefore `db_max_connections` (default 20) + the lock pool (default 40), i.e. - /// 60 by default, and at most `db_max_connections` + 64. Size BOTH against the + /// therefore `db_max_connections` (default 48) + the lock pool (default 40), i.e. + /// 88 by default, and at most `db_max_connections` + 64. Size BOTH against the /// database server's `max_connections`: `db_max_connections`' own doc predates the /// lock pool and no longer covers most of the node's connections. The +8 headroom /// is shared with the three non-push `acquire_write` callers (`api/issues.rs` x2, @@ -492,10 +492,15 @@ pub struct Config { /// absent with a 404. The handler still short-circuits the moment it serves. /// Must be between 1 and 1_048_576. Default: 64. /// - /// The effective ceiling is `max(MAX_PIN_SOURCES + 1, this)`. That floor exists - /// so the cap can never truncate a request before its whole bounded provenance - /// source set has been tried, which would falsely 503 a provenanced request, so - /// setting this below the floor widens nothing and is silently raised. + /// The effective per-request ceiling is the TIGHTER of this knob and the node's + /// internal per-request history-walk ceiling, `MAX_PIN_SOURCES + 1` = 17 (see + /// `api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST` and the `min()` that combines the + /// two in the resolver). Setting this above 17 changes nothing, because the + /// internal ceiling already binds. Setting it below 17 does lower the cap: the + /// constant side of the `min()` is what keeps a request from being truncated + /// before its whole bounded provenance source set has been tried, so an operator + /// who goes under it is choosing a tighter cap that can 503 a provenanced + /// request, which is allowed. #[arg( long, env = "GITLAWB_IPFS_MAX_REPOS_WALKED", diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 70a32874..63ea549e 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -852,10 +852,13 @@ impl Drop for RepoWriteGuard { /// drop: the lock clears shortly after the connection goes away, not /// synchronously with it. /// * It is a SEPARATE pool from the main query pool, not a slice of it. A push -/// holds its lock connection for the whole receive-pack, and -/// `db_max_connections` (default 20) is well below -/// `max_concurrent_git_pushes` (default 32), so drawing these from the main -/// pool would starve every other query during a push burst. +/// holds its lock connection for the whole receive-pack, so drawing these +/// from the main pool would let a burst of `max_concurrent_git_pushes` +/// pushes park that many query connections for the length of their +/// receive-packs and starve every other query. That is true at any pool +/// size, so the separation does not rest on how the two knobs are set; +/// `Config::validate` separately requires `db_max_connections` to clear +/// `max_concurrent_git_pushes` by `DB_POOL_APP_HEADROOM`. /// /// `acquire_timeout` bounds the wait when every lock-pool connection is busy, so /// exhaustion surfaces as a clean error rather than an unbounded hang. diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 83d423c9..bd1233f8 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -65,7 +65,7 @@ struct DbStartupStatus { /// pool used to derive its size straight from that knob, so raising the push cap /// silently raised the node's Postgres connection ceiling with no CLI error and no /// relation to the server's own `max_connections` (#173 F4). The node's total budget is -/// now bounded: `db_max_connections` (default 20) + at most this. +/// now bounded: `db_max_connections` (default 48) + at most this. const LOCK_POOL_MAX_CONNECTIONS: u32 = 64; /// Connections the lock pool keeps above the push cap. Covers the three non-push @@ -313,11 +313,14 @@ async fn main() -> Result<()> { None }; - // Repo write locks run on their own pool, never the main query pool: each - // push holds its connection for the whole receive-pack, and - // db_max_connections (20) is below max_concurrent_git_pushes (32), so sharing - // would starve every other query under a push burst. See build_lock_pool for - // the cancellation semantics (#173). + // Repo write locks run on their own pool, never the main query pool: each push + // holds its connection for the whole receive-pack, so a burst of concurrent + // pushes drawing from the main pool would park that many connections for the + // duration of their receive-packs and starve every other query. That holds + // whatever the two pools are sized at, which is why the separation is + // structural rather than a consequence of the defaults; config validate() + // separately requires db_max_connections >= max_concurrent_git_pushes + 8. See + // build_lock_pool for the cancellation semantics (#173). let lock_pool = git::repo_store::build_lock_pool( db.pool(), lock_pool_size(config.max_concurrent_git_pushes), diff --git a/docs/RUN-A-NODE.md b/docs/RUN-A-NODE.md index e5ad1d91..bcfca17d 100644 --- a/docs/RUN-A-NODE.md +++ b/docs/RUN-A-NODE.md @@ -187,6 +187,8 @@ GITLAWB_ENFORCE_OWNER_PUSH=true **Node refuses to start with "strict-mode operator check failed"** — either `gl node register` first, or unset `GITLAWB_OPERATOR_STRICT_MODE`. +**Node refuses to start with "GITLAWB_DB_MAX_CONNECTIONS (20) must be at least max_concurrent_git_pushes (32) + 8 headroom"**: raise `GITLAWB_DB_MAX_CONNECTIONS` to at least the push cap plus 8 (40 with the default cap of 32; 48 is the shipped default and the recommended value), or lower `GITLAWB_MAX_CONCURRENT_GIT_PUSHES`. This bites a node upgraded in place that still sets the old pool size of 20. The check is deliberate: each concurrent push pins one pooled connection for its whole receive-pack, so a pool that does not clear the push cap lets a burst of slow pushes starve every other database path. + **Rewards are 0 after a week** — run `gl node onchain-status`. If `currentlyActive: false`, check your heartbeat loop (node logs for `operator heartbeat sent`). **Want to rotate operator wallet** — requires unstake → re-register with new wallet. No in-place rotation in v1. From f839fb924653f1b012c2b34d51260ea3faba7f62 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 10 Aug 2026 01:34:04 -0500 Subject: [PATCH 31/77] fix(node): stop reporting a Kubo pin whose DB record never landed On record_pinned_cid_with_source exhausting its retries, pin_new_objects warned and then pushed the object onto the returned vector anyway, so the post-push log told operators an object was pinned when the durable index had no row for it and the resolver could not serve that CID. The push is now conditional on the record succeeding. The twins deliberately diverge here and the old comment said the opposite, that the push mirrored pinata to keep them structurally identical. The Kubo return is log-only, so dropping a record-failed pin costs nothing. The pinata return is a real input: api/repos.rs builds the sha-to-cid map from it, which drives upsert_branch_cid and the p2p ref-update gossip, so omitting rows there would silently drop branch-to-CID records. Both comments now say so, since the next reader's instinct will be to re-align them. Test asserts the record-failed object is absent from the return while the add mock still fired, and that a second object is still attempted rather than the batch breaking. Verified RED against the unconditional push before the fix, and load-bearing by re-injecting it. --- crates/gitlawb-node/src/ipfs_pin.rs | 45 +++++++++++------ crates/gitlawb-node/src/test_support.rs | 65 +++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 14 deletions(-) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 589e5c1f..e526e0ea 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -616,10 +616,14 @@ pub(crate) fn batch_budget_gate( /// the same bounded, reaped git read. It still has no per-request override, since /// `pinata::pin_object` takes no timeout argument and its uploads are bounded by /// the shared client's own ceiling. Everything else about the shape (the -/// skip-if-pinned check, the provenance recording, the fault arms, the returned -/// pairs) changes in lockstep. +/// skip-if-pinned check, the provenance recording, the fault arms) changes in +/// lockstep. The returned pairs are the one deliberate exception: this side omits +/// an object whose DB record exhausted its retries, because the return here is +/// consumed for logging only, while the pinata side still returns it because its +/// return feeds the announcement `cid_map`. See the record step for the reasoning. /// -/// Returns a list of `(sha256_hex, cid)` pairs for objects pinned this call. +/// Returns a list of `(sha256_hex, cid)` pairs pinned AND durably recorded this +/// call. // Eight because #173's git seam (`git_bin`, `git_timeout`) and pin provenance // (`repo_id`) sit alongside #174's batch budget. All four callers pass every one, and // grouping them into a context struct would add a type whose only job is to be @@ -812,19 +816,32 @@ pub async fn pin_new_objects( // resolver. U3 (#173): the pin and its source go down in ONE transaction. // As two independent best-effort calls this path could land the pin while // dropping its own source, producing a source set silently missing its - // first pinner; atomically there is no such window, and a total failure - // leaves the object unpinned so the next push retries the whole thing. - if let Err(e) = - retry_db_record(|| db.record_pinned_cid_with_source(&sha, &raw_cid, repo_id)) - .await + // first pinner; atomically there is no such window. When the transaction + // still fails after every retry, Kubo is holding the bytes but the DB has + // no row, so nothing can resolve that CID and there is no partial state to + // clean up. Recovery is the next push, which re-offers the object and + // retries the whole record; until then the object counts as unpinned, and + // the returned vector says so by carrying only durably recorded pins. + // + // Returning the provider Hash rather than the resolver key is deliberate: + // the DB `cid` is the raw resolver key (recorded above), the returned value + // is the provider CID. On the record-failed case the twins DIVERGE and must + // stay that way. This return is log-only (`api/repos.rs` turns it into a + // count log plus one line per pair and consumes it nowhere else), so + // dropping a record-failed pin costs nothing and stops the log claiming a + // pin the resolver cannot serve. The pinata twin keeps its unconditional + // push because ITS return is a real input: `api/repos.rs` builds the + // sha-to-cid `cid_map` from it, which drives `upsert_branch_cid` and the + // p2p `publish_ref_update` gossip CID. Do not re-align them without moving + // that consumer first. + match retry_db_record(|| db.record_pinned_cid_with_source(&sha, &raw_cid, repo_id)) + .await { - tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); + Ok(()) => pinned.push((sha, cid)), + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); + } } - // Return the provider Hash (not the resolver key), mirroring the pinata - // twin's contract: the DB `cid` is the raw resolver key (recorded above), - // the returned value is the provider CID. Here the return is consumed only - // for logging, but keeping the twins structurally identical avoids drift. - pinned.push((sha, cid)); } Ok(_) => {} Err(e) => { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index edc027d3..0d3959ce 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -4963,6 +4963,71 @@ mod tests { ); } + /// U4 (#173, finding 5): a pin whose DB record exhausts its retries must NOT appear + /// in the returned vector. Kubo really is holding the bytes (the `/add` mock is hit), + /// but with no `pinned_cids` row the resolver cannot serve that CID, so reporting it + /// as pinned overclaims. The Kubo return is log-only (`api/repos.rs` counts the pairs + /// and logs each one), which is what makes omitting the row safe here; the pinata + /// twin's return feeds the announcement `cid_map` and keeps its own contract. + /// + /// Two objects, because `with_pin_sources_broken` hides the table process-wide and + /// the harness cannot express per-object DB breakage. Both records fail, and the + /// `/add` mock being hit exactly twice is the batch-survival proof: the first + /// failure warns and continues instead of breaking out of the loop. The healthy + /// direction (a successful record IS returned) is already covered by + /// `pin_new_objects_records_provenance` directly above, so the two together cover + /// both sides without new harness machinery. + #[sqlx::test] + async fn pin_new_objects_omits_objects_whose_db_record_failed(pool: PgPool) { + let state = test_state(pool.clone()).await; + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyproviderhash"}"#) + .expect(2) + .create_async() + .await; + + let fx = seed_cid_repos("provpin_u4", "ppu4", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("provpin_u4") + .join("pinsrc.git"); + + let pinned = with_pin_sources_broken(&pool, || async { + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone(), fx.secret_oid.clone()], + &state.db, + "repoU4", + // Far above the ~150ms per object the retry ladder spends + // (PIN_RECORD_ATTEMPTS x PIN_RECORD_BACKOFF), so the batch budget gate + // is never what truncates this run. + std::time::Duration::from_secs(60), + ) + .await + }) + .await; + + assert!( + pinned.is_empty(), + "a pin with no durable index row must not be reported as pinned, got {pinned:?}" + ); + // Exactly two adds: the first record failure did not break the batch. + m.assert_async().await; + for oid in [&fx.public_oid, &fx.secret_oid] { + assert_eq!( + state.db.provenance_for_oid(oid).await.unwrap(), + None, + "the record really did fail, so there is no row to report" + ); + } + } + /// #173 (grok F2): the post-push pin read is BOUNDED, so a wedged/D-state /// `git cat-file` (stuck NFS/Tigris backend) is reaped at `git_timeout` and /// `pin_new_objects` RETURNS — reaching `requeue_or_release` in production — From 4681f54327b58b6ebdd569a1cec17a3c2643ad13 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 10 Aug 2026 02:14:10 -0500 Subject: [PATCH 32/77] fix(node): bring the Pinata pin path back into lockstep with the Kubo one Two gaps remained after the main merge closed the unbounded-read one. The Kubo skip branch repairs a legacy provider-CID row to the raw-content resolver key on re-push and the Pinata skip branch did not, so a Pinata-first node left those rows unresolvable until the deferred sweep caught them. And the Pinata object read was bounded only by the batch deadline, so one wedged cat-file could hold the pin permit for the whole budget. Both are fixed by threading git_timeout into pinata::pin_new_objects, which it needed anyway: repair_legacy_provider_cid takes a Duration, and the read bound is now min(batch deadline, now + git_timeout), the same shape the Kubo side uses. repair_legacy_provider_cid becomes pub(crate). The returned pairs still diverge on purpose and the twin docs say why: Kubo omits a record-failed pin because its return is log-only, Pinata keeps it because its return feeds the announcement cid_map. Pinata's return behavior is unchanged. Four tests drive pinata::pin_new_objects rather than the Kubo twin, because a repair call merely present in pinata.rs proves nothing about the Pinata lane: the repair runs on the skip branch, the codec cost gate reads no bytes for a row that is already canonical, a failed repair does not break the batch, and the upload path never runs the repair. The two must-not guards have no obtainable pre-fix RED, so they are proven by mutation instead: removing the cost gate and moving the repair onto the upload path each turn one red. --- crates/gitlawb-node/src/api/repos.rs | 1 + crates/gitlawb-node/src/ipfs_pin.rs | 24 +- crates/gitlawb-node/src/pinata.rs | 130 +++++++++- crates/gitlawb-node/src/test_support.rs | 329 ++++++++++++++++++++++++ 4 files changed, 460 insertions(+), 24 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 4298e1d8..ffed6257 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2715,6 +2715,7 @@ async fn post_receive_replication_tail( // The literal, not `state.git_bin`: tests point that at a fake // walk git, and this read must run the real one. "git", + pinata_git_timeout, object_list, &db_clone, &repo_id, diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index e526e0ea..caa61585 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -60,7 +60,7 @@ where /// CIDv1/raw key is already the resolver key and reads NO bytes, keeping the /// steady-state skip cost DB-only. Only a legacy-codec row reads the object to /// recompute. A row whose bytes are gone stays withheld (no destructive rewrite). -async fn repair_legacy_provider_cid( +pub(crate) async fn repair_legacy_provider_cid( repo_path: &std::path::Path, git_bin: &str, git_timeout: Duration, @@ -611,16 +611,18 @@ pub(crate) fn batch_budget_gate( /// on the repo takes the full-scan fallback (`push_delta::list_all_objects`) and /// re-derives the whole object set, which then re-offers the skipped OIDs. /// -/// The twin in `pinata.rs` is back at parity on the two things that bound a -/// batch: it runs the same shared budget gate at the top of every iteration and -/// the same bounded, reaped git read. It still has no per-request override, since -/// `pinata::pin_object` takes no timeout argument and its uploads are bounded by -/// the shared client's own ceiling. Everything else about the shape (the -/// skip-if-pinned check, the provenance recording, the fault arms) changes in -/// lockstep. The returned pairs are the one deliberate exception: this side omits -/// an object whose DB record exhausted its retries, because the return here is -/// consumed for logging only, while the pinata side still returns it because its -/// return feeds the announcement `cid_map`. See the record step for the reasoning. +/// The twin in `pinata.rs` is back at parity on everything that bounds or repairs an +/// object: it runs the same shared budget gate at the top of every iteration, the same +/// bounded and reaped git read against the earlier of the batch deadline and +/// `git_timeout`, and the same opportunistic legacy provider-CID repair on its skip +/// branch. It still has no per-request override, since `pinata::pin_object` takes no +/// timeout argument and its uploads are bounded by the shared client's own ceiling. +/// Everything else about the shape (the skip-if-pinned check, the provenance recording, +/// the fault arms) changes in lockstep. The returned pairs are the one deliberate +/// exception: this side omits an object whose DB record exhausted its retries, because +/// the return here is consumed for logging only, while the pinata side still returns it +/// because its return feeds the announcement `cid_map`. See the record step for the +/// reasoning. /// /// Returns a list of `(sha256_hex, cid)` pairs pinned AND durably recorded this /// call. diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index c2473c2e..646fa59f 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -76,6 +76,8 @@ pub async fn pin_object( /// still needed to read each object's bytes, and `git_bin` names the binary those /// reads run: the production caller passes the literal `"git"`, and a test passes a /// fake so the loop's own bound can be driven with a git that never answers. +/// `git_timeout` is the per-object read bound, the same value and the same role it has +/// in the twin: it bounds both the pin read and the skip branch's opportunistic repair. /// Objects already recorded with a `pinata_cid` are skipped, and `repo_id` records the /// pin's provenance (#173). Returns `(sha_hex, provider_cid)` pairs for each newly /// pinned object: the provider CID is the Pinata gateway CID (used for branch->CID @@ -93,11 +95,12 @@ pub async fn pin_object( /// than the read floor left. It is a gate, not a hard ceiling, since a started /// iteration still runs to completion; /// - the git read: `store::read_object_bounded` runs under `spawn_blocking` against the -/// ABSOLUTE batch deadline (not the loop-top remainder, which the `has_pinata_cid` -/// round-trip sitting between the two would push past it), with SIGTERM-then-SIGKILL -/// process-group teardown, so a hung `git cat-file` costs this batch its remaining -/// budget plus one watchdog teardown instead of holding the permit for the child's -/// whole lifetime and blocking a runtime worker while it does. +/// earlier of the ABSOLUTE batch deadline (not the loop-top remainder, which the +/// `has_pinata_cid` round-trip sitting between the two would push past it) and this +/// object's own `git_timeout`, with SIGTERM-then-SIGKILL process-group teardown, so a +/// hung `git cat-file` costs this batch one `git_timeout` plus one watchdog teardown +/// instead of holding the permit for the child's whole lifetime and blocking a runtime +/// worker while it does. /// /// So the LOOP's hold is bounded by roughly `batch_budget`, plus one watchdog /// teardown and one upload (the shared client's whole-request timeout bounds the @@ -108,13 +111,23 @@ pub async fn pin_object( /// (`has_pinata_cid`, `record_pinata_cid`) are untimed inside the budgeted region /// too. /// -/// The twin in `ipfs_pin.rs` shares the budget gate and the bounded read with this -/// loop, so both are at parity again. Change them in lockstep: the skip-if-pinned -/// check, the fault arms, the returned pairs, and the budget handling. -// Nine arguments, over clippy's threshold: the two the budget adds (`git_bin`, -// `batch_budget`) plus #173's `repo_id` are what put the read under test injection and under a deadline, and -// grouping them into a struct would only move the same values behind a name the twin in -// `ipfs_pin.rs` does not use. Same allow as the sibling call sites in `api::repos`. +/// The twin in `ipfs_pin.rs` is at parity with this loop on everything that bounds or +/// repairs an object: the shared budget gate, the read bounded by the earlier of the +/// batch deadline and `git_timeout`, and the skip branch's opportunistic legacy +/// provider-CID repair. Change them in lockstep: the skip-if-pinned check, the +/// provenance and source recording, the fault arms, and the budget handling. +/// +/// The RETURNED PAIRS are the one deliberate divergence, and it is not drift. This side +/// pushes a pin whose DB record exhausted its retries, because this return is a real +/// input: `api::repos` builds the sha-to-cid `cid_map` from it, which drives +/// `upsert_branch_cid` and the p2p `publish_ref_update` gossip CID. The twin's return is +/// log-only, so it omits a record-failed pin rather than logging a pin the resolver +/// cannot serve. Moving this side to match would need that consumer moved first. +// Ten arguments, over clippy's threshold: the three the budget and the git seam add +// (`git_bin`, `git_timeout`, `batch_budget`) plus #173's `repo_id` are what put the read +// under test injection and under a deadline, and grouping them into a struct would only +// move the same values behind a name the twin in `ipfs_pin.rs` does not use. Same allow +// as the sibling call sites in `api::repos`. #[allow(clippy::too_many_arguments)] pub async fn pin_new_objects( client: &reqwest::Client, @@ -122,6 +135,7 @@ pub async fn pin_new_objects( jwt: &str, repo_path: &std::path::Path, git_bin: &str, + git_timeout: Duration, object_list: Vec, db: &crate::db::Db, repo_id: &str, @@ -180,6 +194,23 @@ pub async fn pin_new_objects( tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); } } + // R8 (#173 round 10), in lockstep with the ipfs_pin skip branch: + // opportunistically repair a legacy provider-CID row (Kubo dag-pb / + // Pinata) to the raw-content resolver key on this re-push. Cost-gated on + // the stored key's codec, so a non-legacy row reads no bytes. Warn-only: + // a failure leaves the row as-is for a later re-push or the deferred + // one-shot sweep. + if let Err(e) = crate::ipfs_pin::repair_legacy_provider_cid( + repo_path, + git_bin, + git_timeout, + &sha, + db, + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to repair legacy provider CID"); + } continue; } Ok(false) => {} @@ -199,7 +230,13 @@ pub async fn pin_new_objects( // between the two, so `Instant::now() + budget_left` would land past `deadline` by // however long the DB took, and under a saturated pool that is the dominant term. // A slow DB check must not push the read's own bound out. - let read_deadline = deadline; + // + // Bounded by the EARLIER of the batch deadline (#174) and this object's own + // `git_timeout` (#173), the same pair the ipfs_pin twin uses. Both bounds are + // load-bearing and neither implies the other: the batch deadline alone would let + // ONE wedged `cat-file` hold the pin permit for the whole budget, while + // `git_timeout` alone would let a batch of merely-slow reads run past the budget. + let read_deadline = std::cmp::min(deadline, std::time::Instant::now() + git_timeout); let read_path = repo_path.to_path_buf(); let read_sha = sha.clone(); let read_git = git_bin.to_string(); @@ -492,6 +529,7 @@ mod tests { "test-jwt", &repo_path, "git", + Duration::from_secs(60), oids, &db, "repo-merge-test", @@ -582,6 +620,7 @@ mod tests { "test-jwt", &repo_path, fake.to_str().unwrap(), + Duration::from_secs(60), oids, &db, "repo-merge-test", @@ -635,6 +674,66 @@ mod tests { ); } + /// U3 scenario 5 (#173): the read is bounded by the EARLIER of the batch deadline and + /// this object's own `git_timeout`, the same pair the ipfs_pin twin uses. The batch + /// budget here is generous (60s) so the budget gate cannot be what ends the call: only + /// the 1s `git_timeout` can. A wedged `git cat-file` that traps SIGTERM and sleeps 30s + /// must therefore be reaped in the `git_timeout` order and the call must return, rather + /// than holding the pin permit for the whole budget. + /// + /// RED with `let read_deadline = deadline;` (the pre-U3 bare batch deadline): the read + /// waits out the wedged child, the call runs ~30s, and the outer 20s timeout fires. + #[cfg(unix)] + #[sqlx::test] + async fn pin_new_objects_bounds_the_read_by_git_timeout_not_the_batch_budget( + pool: sqlx::PgPool, + ) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("git-timeout.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let fake = tmp.path().join("hanging-git"); + write_script(&fake, "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nsleep 30\n"); + + let (_logs, _guard) = capture_logs(); + let client = reqwest::Client::new(); + let started = std::time::Instant::now(); + let pinned = tokio::time::timeout( + Duration::from_secs(20), + pin_new_objects( + &client, + "http://127.0.0.1:9", + "test-jwt", + &repo_path, + fake.to_str().unwrap(), + // The bound under test. + Duration::from_secs(1), + oids, + &db, + "repo-git-timeout", + // Generous, so a call that ends on time ended on `git_timeout`. + Duration::from_secs(60), + ), + ) + .await + .expect( + "the read must be bounded by git_timeout, not by the batch budget: a wedged git \ + cannot hold the pin permit for the whole 60s budget", + ); + let elapsed = started.elapsed(); + + assert!( + pinned.is_empty(), + "a git that never answers cannot produce a pinned object: {pinned:?}" + ); + assert!( + elapsed < Duration::from_secs(15), + "elapsed {elapsed:?} must stay in the git_timeout order (1s plus one watchdog \ + teardown), not the 60s batch budget" + ); + } + /// A `git_bin` wrapper that records every invocation's arguments and then execs the /// real git, so a test can tell which objects the loop actually attempted. The returned /// pin list cannot: it is empty both when the loop broke after one object and when it @@ -711,6 +810,7 @@ mod tests { "test-jwt", &repo_path, &git_bin, + Duration::from_secs(60), oids, &db, "repo-merge-test", @@ -784,6 +884,7 @@ mod tests { "test-jwt", &repo_path, &git_bin, + Duration::from_secs(60), oids, &db, "repo-merge-test", @@ -834,6 +935,7 @@ mod tests { "test-jwt", &repo_path, "git", + Duration::from_secs(60), oids.clone(), &db, "repo-merge-test", @@ -862,6 +964,7 @@ mod tests { "test-jwt", &repo_path, "git", + Duration::from_secs(60), oids, &db, "repo-merge-test", @@ -914,6 +1017,7 @@ mod tests { "", &repo_path, fake.to_str().unwrap(), + Duration::from_secs(60), oids, &db, "repo-merge-test", diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 0d3959ce..763371de 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -4346,6 +4346,9 @@ mod tests { "test-jwt", &bare, "git", + // Generous: this test measures the record retry backoff, not the + // read bound, so the git_timeout must never be what fires. + std::time::Duration::from_secs(60), vec![oid], &state.db, &repo_id, @@ -4393,6 +4396,332 @@ mod tests { ); } + /// A Pinata upload mock that must never fire, for the skip-branch tests below: an + /// object already carrying a `pinata_cid` is skipped before the upload, so a call + /// here means the branch under test was not the one taken. + async fn pinata_upload_mock_never(server: &mut mockito::ServerGuard) -> mockito::Mock { + server + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .with_body(r#"{"data":{"cid":"QmShouldNotHappen"}}"#) + .expect(0) + .create_async() + .await + } + + /// The raw-content resolver key for an object in a bare repo, computed the way the + /// pin path computes it. + fn raw_key_for(bare: &std::path::Path, oid: &str) -> String { + gitlawb_core::cid::Cid::from_git_object_bytes( + &crate::git::store::read_object(bare, oid) + .expect("read object bytes") + .expect("object exists") + .1, + ) + .to_string() + } + + /// Seed a `pinned_cids` row by hand: the production helpers always store the raw + /// key, so a legacy provider-CID row can only be written with raw SQL. + async fn seed_pinned_row( + pool: &PgPool, + oid: &str, + cid: &str, + pinata_cid: Option<&str>, + repo_id: &str, + ) { + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid, repo_id) + VALUES ($1, $2, $3, $4, $5)", + ) + .bind(oid) + .bind(cid) + .bind("2020-01-01T00:00:00Z") + .bind(pinata_cid) + .bind(repo_id) + .execute(pool) + .await + .expect("seed pinned_cids row"); + } + + /// The stored resolver key and the stashed old provider value for a pinned object. + async fn stored_key_and_stash(pool: &PgPool, oid: &str) -> (String, Option) { + sqlx::query_as("SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1") + .bind(oid) + .fetch_one(pool) + .await + .expect("the pinned row exists") + } + + /// U3 scenario 1 (#173, Finding 2 lockstep): the PINATA skip branch runs the same + /// opportunistic legacy provider-CID repair the ipfs_pin skip branch runs. A row keyed + /// on a legacy provider CID that already carries a `pinata_cid` (so `has_pinata_cid` + /// answers true and the skip branch is taken) is rewritten to the raw-content resolver + /// key, stashing the old provider value in `legacy_provider_cid`. + /// + /// This drives `pinata::pin_new_objects`, never the ipfs_pin twin: the repair call + /// being PRESENT in `pinata.rs` proves nothing, only its execution through this lane + /// does. RED before the skip-branch call lands (the key stays the provider CID). + #[sqlx::test] + async fn pinata_skip_branch_repairs_legacy_provider_cid(pool: PgPool) { + let state = test_state(pool.clone()).await; + let fx = seed_cid_repos("pinatarepair", "pr", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("pinatarepair") + .join("pinsrc.git"); + + let raw_cid = raw_key_for(&bare, &fx.public_oid); + let provider_cid = legacy_dagpb_cid(&raw_cid); + assert_ne!( + provider_cid, raw_cid, + "the provider CID differs from the raw resolver key" + ); + seed_pinned_row( + &pool, + &fx.public_oid, + &provider_cid, + Some("QmPinataProvider"), + "repoPinataRepair", + ) + .await; + + let mut server = mockito::Server::new_async().await; + let m = pinata_upload_mock_never(&mut server).await; + let client = reqwest::Client::new(); + crate::pinata::pin_new_objects( + &client, + &server.url(), + "test-jwt", + &bare, + "git", + std::time::Duration::from_secs(60), + vec![fx.public_oid.clone()], + &state.db, + "repoPinataRepair", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; + + let (stored_cid, stashed) = stored_key_and_stash(&pool, &fx.public_oid).await; + assert_eq!( + stored_cid, raw_cid, + "the pinata skip branch repairs the key to the raw-content CID" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed in legacy_provider_cid" + ); + } + + /// U3 scenario 2 (#173, cost gate): a canonical raw-CIDv1 row on the pinata skip + /// branch reads NO object bytes. Candidacy is decided from the stored key's codec + /// alone, so the steady-state skip cost stays DB-only on this lane too. The counter + /// lives inside `repair_legacy_provider_cid`, so it counts for whichever lane calls + /// it; this is the both-ways guard (removing the codec gate reads the raw row). + #[sqlx::test] + async fn pinata_skip_branch_repair_codec_gate_skips_raw_row(pool: PgPool) { + let state = test_state(pool.clone()).await; + let fx = seed_cid_repos("pinatagate", "pg", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("pinatagate") + .join("pinsrc.git"); + + let raw_cid = raw_key_for(&bare, &fx.public_oid); + assert!( + gitlawb_core::cid::is_raw_cidv1(&raw_cid), + "the steady-state key is a CIDv1/raw key" + ); + seed_pinned_row( + &pool, + &fx.public_oid, + &raw_cid, + Some("QmPinataProvider"), + "repoPinataGate", + ) + .await; + + let mut server = mockito::Server::new_async().await; + let m = pinata_upload_mock_never(&mut server).await; + let client = reqwest::Client::new(); + crate::ipfs_pin::reset_legacy_repair_reads(); + crate::pinata::pin_new_objects( + &client, + &server.url(), + "test-jwt", + &bare, + "git", + std::time::Duration::from_secs(60), + vec![fx.public_oid.clone()], + &state.db, + "repoPinataGate", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; + + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a CIDv1/raw row triggers no object read on the pinata skip path (cost gate)" + ); + assert_eq!( + state + .db + .cid_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some(raw_cid.as_str()), + "the raw row is left as-is" + ); + } + + /// U3 scenario 3 (#173): a repair that cannot complete is warn-only. It neither + /// aborts the batch nor loses the pin. The first object is a legacy row whose bytes + /// are NOT in the repo, so the read verifies an absence and the row stays withheld + /// rather than being destructively rewritten; the skip branch's own source record + /// still lands for it, and the SECOND object's legacy row is still repaired, which is + /// what proves the batch ran past the failure. + #[sqlx::test] + async fn pinata_skip_branch_repair_failure_is_warn_only(pool: PgPool) { + let state = test_state(pool.clone()).await; + let fx = seed_cid_repos("pinatawarn", "pw", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("pinatawarn") + .join("pinsrc.git"); + + // Object A: bytes absent from this repo, so its repair cannot complete. + let absent_oid = "a".repeat(64); + let absent_provider = legacy_dagpb_cid(&raw_key_for(&bare, &fx.secret_oid)); + seed_pinned_row( + &pool, + &absent_oid, + &absent_provider, + Some("QmPinataProviderA"), + "repoPinataWarn", + ) + .await; + // Object B: a repairable legacy row, queued behind A. + let raw_cid = raw_key_for(&bare, &fx.public_oid); + let provider_cid = legacy_dagpb_cid(&raw_cid); + seed_pinned_row( + &pool, + &fx.public_oid, + &provider_cid, + Some("QmPinataProviderB"), + "repoPinataWarn", + ) + .await; + + let mut server = mockito::Server::new_async().await; + let m = pinata_upload_mock_never(&mut server).await; + let client = reqwest::Client::new(); + crate::pinata::pin_new_objects( + &client, + &server.url(), + "test-jwt", + &bare, + "git", + std::time::Duration::from_secs(60), + vec![absent_oid.clone(), fx.public_oid.clone()], + &state.db, + "repoPinataWarn", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; + + let (a_cid, a_stash) = stored_key_and_stash(&pool, &absent_oid).await; + assert_eq!( + a_cid, absent_provider, + "an unrepairable row is never destructively rewritten" + ); + assert_eq!(a_stash, None, "nothing is stashed for an unrepaired row"); + assert_eq!( + state.db.pin_sources_for_oid(&absent_oid).await.unwrap(), + vec!["repoPinataWarn".to_string()], + "the skip branch's source record still lands: a failed repair loses no pin" + ); + + let (b_cid, b_stash) = stored_key_and_stash(&pool, &fx.public_oid).await; + assert_eq!( + b_cid, raw_cid, + "the batch ran past the failed repair and repaired the later object" + ); + assert_eq!(b_stash.as_deref(), Some(provider_cid.as_str())); + } + + /// U3 scenario 4 (#173, the must-not case): the repair is inside the `has_pinata_cid` + /// skip branch and nowhere else. An object with NO `pinata_cid` takes the upload path, + /// so it must run NO repair read and its (legacy-shaped) key must be left exactly as + /// stored, even though the row would be a repair candidate on the skip branch. Moving + /// the call out of the `Ok(true)` arm reads bytes here and trips the counter. + #[sqlx::test] + async fn pinata_upload_path_never_runs_the_legacy_repair(pool: PgPool) { + let state = test_state(pool.clone()).await; + let fx = seed_cid_repos("pinatanoskip", "pn", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("pinatanoskip") + .join("pinsrc.git"); + + // Legacy-shaped row with NO pinata_cid: `has_pinata_cid` is false, so the skip + // branch is not taken and the object goes to the upload path. + let raw_cid = raw_key_for(&bare, &fx.public_oid); + let provider_cid = legacy_dagpb_cid(&raw_cid); + seed_pinned_row( + &pool, + &fx.public_oid, + &provider_cid, + None, + "repoPinataNoSkip", + ) + .await; + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .with_body(r#"{"data":{"cid":"QmPinataUploaded"}}"#) + .expect(1) + .create_async() + .await; + let client = reqwest::Client::new(); + crate::ipfs_pin::reset_legacy_repair_reads(); + let pinned = crate::pinata::pin_new_objects( + &client, + &server.url(), + "test-jwt", + &bare, + "git", + std::time::Duration::from_secs(60), + vec![fx.public_oid.clone()], + &state.db, + "repoPinataNoSkip", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; + + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "the upload path must never run the skip-branch repair" + ); + let (stored_cid, stashed) = stored_key_and_stash(&pool, &fx.public_oid).await; + assert_eq!( + stored_cid, provider_cid, + "an object that never reached the skip branch keeps its stored key untouched" + ); + assert_eq!(stashed, None, "and nothing is stashed for it"); + assert_eq!( + pinned, + vec![(fx.public_oid.clone(), "QmPinataUploaded".to_string())], + "the pinata return still carries the provider CID for the announcement cid_map" + ); + } + /// U3 scenario 6 (#173, authorization): the marker arms a FALLBACK, never a bypass. /// With the set marked incomplete and the object living only in a repo the caller /// may not read, the scan gates every repo through the same per-caller gate, so the From ea2219b2cb635da61f120f65434db4c427494f6c Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 10 Aug 2026 07:43:53 -0500 Subject: [PATCH 33/77] fix(node): bound the legacy-repair read by the pin batch deadline repair_legacy_provider_cid built its own deadline from git_timeout and never consulted the batch deadline, but both pin loops call it with a pin_semaphore permit held. At shipped defaults that is a 600s git_service_timeout_secs against a 120s PIN_BATCH_BUDGET, so one legacy row whose cat-file wedges could hold a global pin slot for five times the budget the batch is supposed to cost and starve every other repo's pin work. The loop's budget gate cannot help: it runs at the top of the next iteration and cannot preempt a call already in flight. The helper now takes the deadline rather than deriving it, since its two kinds of caller can afford different holds. Both pin loops clamp to min(batch deadline, now + git_timeout). The boot sweep holds no permit and has no batch to overrun, so it keeps the plain git_timeout. The Kubo side has had this since the repair landed and the Pinata side inherited it from the lockstep work in the previous commit, so both get a test: a wedged cat-file with a 60s git_timeout and a 2s batch budget must return on the batch order. Both were RED at 62s before the clamp and are load-bearing by mutation. Also stops the read-bound comments implying both arms bind in a default deployment. At shipped defaults the batch deadline is always the tighter one; the git_timeout arm is what an operator who lowers that knob gets. --- crates/gitlawb-node/src/ipfs_pin.rs | 45 +++++-- crates/gitlawb-node/src/pinata.rs | 8 +- crates/gitlawb-node/src/test_support.rs | 157 ++++++++++++++++++++++++ 3 files changed, 202 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index caa61585..62621b98 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -60,10 +60,18 @@ where /// CIDv1/raw key is already the resolver key and reads NO bytes, keeping the /// steady-state skip cost DB-only. Only a legacy-codec row reads the object to /// recompute. A row whose bytes are gone stays withheld (no destructive rewrite). +/// +/// The read's `deadline` is the CALLER's to set, because the two kinds of caller can +/// afford very different holds. Both pin loops run this while holding a `pin_semaphore` +/// permit, so they clamp it to the batch deadline: left at `git_service_timeout_secs` +/// (600s by default) one wedged `cat-file` would hold a GLOBAL pin slot for five times +/// `PIN_BATCH_BUDGET` and starve every other repo's pin work, and the loop's own budget +/// gate cannot preempt a call already in flight. The boot sweep holds no permit and has +/// no batch to overrun, so it passes the plain `git_timeout`. pub(crate) async fn repair_legacy_provider_cid( repo_path: &std::path::Path, git_bin: &str, - git_timeout: Duration, + deadline: std::time::Instant, sha: &str, db: &crate::db::Db, ) -> Result { @@ -91,9 +99,8 @@ pub(crate) async fn repair_legacy_provider_cid( let git_bin = git_bin.to_string(); let sha = sha.to_string(); // The shared-deadline form: `read_object_bounded` composes its type probe and - // content read under ONE `git_timeout`, rather than granting each stage a full - // one, so a legacy row's repair read is bounded by the configured budget total. - let deadline = std::time::Instant::now() + git_timeout; + // content read under ONE deadline, rather than granting each stage a full budget, + // so a legacy row's repair read is bounded in total by whatever the caller set. tokio::task::spawn_blocking(move || { crate::git::store::read_object_bounded(&git_bin, &repo_path, &sha, deadline) }) @@ -249,7 +256,17 @@ async fn sweep_pass( row_retryable = true; continue; } - match repair_legacy_provider_cid(&repo_path, git_bin, git_timeout, &sha, db).await { + // The sweep holds no pin permit and has no batch to overrun, so the plain + // `git_timeout` is the right budget here. + match repair_legacy_provider_cid( + &repo_path, + git_bin, + std::time::Instant::now() + git_timeout, + &sha, + db, + ) + .await + { Ok(RepairOutcome::Repaired) => { repaired += 1; row_repaired = true; @@ -699,8 +716,17 @@ pub async fn pin_new_objects( // re-push. Cost-gated on the stored key's codec — a non-legacy row // reads no bytes. Warn-only: a failure leaves the row as-is for a // later re-push or the deferred one-shot sweep. - if let Err(e) = - repair_legacy_provider_cid(repo_path, git_bin, git_timeout, &sha, db).await + // Clamped to the batch deadline: this runs with the pin permit held, so + // an unclamped `git_timeout` would let one wedged read hold a global pin + // slot for 600s against a 120s budget. + if let Err(e) = repair_legacy_provider_cid( + repo_path, + git_bin, + std::cmp::min(deadline, std::time::Instant::now() + git_timeout), + &sha, + db, + ) + .await { tracing::warn!(sha = %sha, err = %e, "failed to repair legacy provider CID"); } @@ -732,6 +758,11 @@ pub async fn pin_new_objects( // other: the batch deadline alone would let ONE wedged `cat-file` hold the pin // permit for the whole 120s budget (the failure #173's reaper test drives), while // `git_timeout` alone would let a batch of merely-slow reads run past the budget. + // Which arm actually binds depends on configuration, and at SHIPPED DEFAULTS it is + // always the batch deadline: `git_service_timeout_secs` is 600 against a 120s + // PIN_BATCH_BUDGET. The `git_timeout` arm is what an operator who tightens that + // knob below the remaining budget gets, so do not read this as two bounds both + // firing in a default deployment. let read_deadline = std::cmp::min(deadline, std::time::Instant::now() + git_timeout); let read_path = repo_path.to_path_buf(); let read_sha = sha.clone(); diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 646fa59f..da76e503 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -200,10 +200,13 @@ pub async fn pin_new_objects( // the stored key's codec, so a non-legacy row reads no bytes. Warn-only: // a failure leaves the row as-is for a later re-push or the deferred // one-shot sweep. + // Clamped to the batch deadline, in lockstep with the ipfs_pin twin: this + // runs with the pin permit held, so an unclamped `git_timeout` would let + // one wedged read hold a global pin slot for 600s against a 120s budget. if let Err(e) = crate::ipfs_pin::repair_legacy_provider_cid( repo_path, git_bin, - git_timeout, + std::cmp::min(deadline, std::time::Instant::now() + git_timeout), &sha, db, ) @@ -236,6 +239,9 @@ pub async fn pin_new_objects( // load-bearing and neither implies the other: the batch deadline alone would let // ONE wedged `cat-file` hold the pin permit for the whole budget, while // `git_timeout` alone would let a batch of merely-slow reads run past the budget. + // As on the twin, at SHIPPED DEFAULTS the batch deadline is the arm that binds + // (600s git timeout against a 120s budget); the `git_timeout` arm is for an + // operator who tightens that knob below the remaining budget. let read_deadline = std::cmp::min(deadline, std::time::Instant::now() + git_timeout); let read_path = repo_path.to_path_buf(); let read_sha = sha.clone(); diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 763371de..30d6325f 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -4453,6 +4453,163 @@ mod tests { .expect("the pinned row exists") } + /// The skip-branch repair runs while the caller holds a `pin_semaphore` permit, so it + /// must be bounded by the BATCH deadline and not by `git_timeout` alone. + /// `repair_legacy_provider_cid` builds its own deadline, and at shipped defaults that + /// is `git_service_timeout_secs` (600s) against a `PIN_BATCH_BUDGET` of 120s: one + /// legacy row whose `cat-file` wedges would hold a GLOBAL pin slot for five times the + /// budget the batch is supposed to cost, starving every other repo's pin work. The + /// loop's own budget gate cannot help, since it only runs at the top of the NEXT + /// iteration and cannot preempt a call already in flight. + /// + /// A wedged `cat-file`, a generous 60s `git_timeout`, and a 2s batch budget: the call + /// must return on the batch order. Both pin-permit-holding callers of the repair share + /// this clamp; the boot sweep keeps the plain `git_timeout`, since it holds no permit + /// and has no batch to overrun. + /// + /// REVERT PROOF (RED): pass `Instant::now() + git_timeout` to the repair instead of the + /// batch-clamped deadline and the wedged child runs the full 60s, blowing the outer + /// timeout below. + #[cfg(unix)] + #[sqlx::test] + async fn pinata_skip_branch_repair_is_bounded_by_the_batch_deadline(pool: PgPool) { + let state = test_state(pool.clone()).await; + let fx = seed_cid_repos("pinatabound", "pb", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("pinatabound") + .join("pinsrc.git"); + + // Resolve the real key with the real git BEFORE the fake is wired in, so the row + // is genuinely legacy-shaped and the repair has real work to attempt. + let raw_cid = raw_key_for(&bare, &fx.public_oid); + let provider_cid = legacy_dagpb_cid(&raw_cid); + seed_pinned_row( + &pool, + &fx.public_oid, + &provider_cid, + Some("QmPinataProvider"), + "repoPinataBound", + ) + .await; + + // `cat-file` never answers and ignores SIGTERM, so only the watchdog's group + // SIGKILL at the deadline can end it. Which deadline that is, is the whole test. + let tmp = tempfile::TempDir::new().unwrap(); + let fake = tmp.path().join("wedged-git"); + std::fs::write( + &fake, + "#!/bin/sh\ntrap '' TERM\ncase \"$1\" in\n cat-file) sleep 60 ;;\n *) : ;;\nesac\nexit 0\n", + ) + .unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + } + + let mut server = mockito::Server::new_async().await; + let m = pinata_upload_mock_never(&mut server).await; + let client = reqwest::Client::new(); + let started = std::time::Instant::now(); + tokio::time::timeout( + std::time::Duration::from_secs(25), + crate::pinata::pin_new_objects( + &client, + &server.url(), + "test-jwt", + &bare, + fake.to_str().unwrap(), + // Generous: if the call ends on time it ended on the batch deadline. + std::time::Duration::from_secs(60), + vec![fx.public_oid.clone()], + &state.db, + "repoPinataBound", + // The bound under test. + std::time::Duration::from_secs(2), + ), + ) + .await + .expect( + "a wedged skip-branch repair must be reaped on the batch deadline, not held for \ + the whole git_timeout while it pins a global pin permit", + ); + m.assert_async().await; + + let elapsed = started.elapsed(); + assert!( + elapsed < std::time::Duration::from_secs(20), + "elapsed {elapsed:?} must stay in the 2s batch-budget order (plus one watchdog \ + teardown), not the 60s git_timeout order" + ); + } + + /// The ipfs_pin twin of the clamp above, and the one that has been shipping: the Kubo + /// skip branch has always called the repair with a bare `git_timeout` while holding the + /// pin permit. Same wedged `cat-file`, same 60s `git_timeout` against a 2s batch + /// budget, same requirement that the call return on the batch order. + /// + /// REVERT PROOF (RED): drop the `min(deadline, ...)` clamp at the ipfs_pin skip-branch + /// call and this blows its outer timeout. + #[cfg(unix)] + #[sqlx::test] + async fn kubo_skip_branch_repair_is_bounded_by_the_batch_deadline(pool: PgPool) { + let state = test_state(pool.clone()).await; + let fx = seed_cid_repos("kubobound", "kb", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("kubobound") + .join("pinsrc.git"); + + let raw_cid = raw_key_for(&bare, &fx.public_oid); + let provider_cid = legacy_dagpb_cid(&raw_cid); + // A row in pinned_cids makes `is_pinned` true, so the Kubo loop takes the skip + // branch and reaches the repair without ever attempting an add. + seed_pinned_row(&pool, &fx.public_oid, &provider_cid, None, "repoKuboBound").await; + + let tmp = tempfile::TempDir::new().unwrap(); + let fake = tmp.path().join("wedged-git"); + std::fs::write( + &fake, + "#!/bin/sh\ntrap '' TERM\ncase \"$1\" in\n cat-file) sleep 60 ;;\n *) : ;;\nesac\nexit 0\n", + ) + .unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + } + + let started = std::time::Instant::now(); + tokio::time::timeout( + std::time::Duration::from_secs(25), + crate::ipfs_pin::pin_new_objects( + // Empty endpoint would return before the loop, so point at a closed port: + // the skip branch is reached and no add is ever attempted anyway. + "http://127.0.0.1:9", + &bare, + fake.to_str().unwrap(), + std::time::Duration::from_secs(60), + vec![fx.public_oid.clone()], + &state.db, + "repoKuboBound", + std::time::Duration::from_secs(2), + ), + ) + .await + .expect( + "a wedged skip-branch repair must be reaped on the batch deadline, not held for \ + the whole git_timeout while it pins a global pin permit", + ); + + let elapsed = started.elapsed(); + assert!( + elapsed < std::time::Duration::from_secs(20), + "elapsed {elapsed:?} must stay in the 2s batch-budget order, not the 60s \ + git_timeout order" + ); + } + /// U3 scenario 1 (#173, Finding 2 lockstep): the PINATA skip branch runs the same /// opportunistic legacy provider-CID repair the ipfs_pin skip branch runs. A row keyed /// on a legacy provider CID that already carries a `pinata_cid` (so `has_pinata_cid` From bad82b3cd9fa970ebf67817395bfbd98591879b1 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:29:25 -0500 Subject: [PATCH 34/77] fix(node): clamp every admission-held DB await in the CID resolver get_by_cid builds its walk admission (the global permit plus the per-caller permit) and then made several DB awaits with no deadline while those scarce permits were held. A stalled pool or a blocked query pinned every walk slot for the whole stall, so later /ipfs requests shed capacity 503s long after GITLAWB_IPFS_REQUEST_BUDGET_SECS had elapsed. Any unauthenticated caller can reach it. The clamped set is derived from the admission-held region itself, not from the two sites the review named. Walking the handler body found four bare awaits: oids_for_cid, pin_sources_for_oid, pin_sources_at_cap, pin_sources_incomplete The last two were not named in review. Already clamped and left alone: the three per-source lookups and the legacy-scan preload trio. Two negatives worth recording: gate_and_serve is the only callee reached from inside the region and holds no DB awaits of its own, so the region does not extend transitively, and ipfs_work_rate_limiter.is_throttled is in-memory, not a round trip. Ten DB awaits in the body, zero bare after this change. Each clamp takes the arm shape the per-source lookups already use, so a timeout returns the budget-tainted retryable 503 and the early return RAII-drops the admission. budget_shed and remaining are hoisted above the oids_for_cid await so the provenance-path sites share one definition; the legacy-scan preload keeps its own in its nested scope. Tests: four stall regressions, one per clamped site. The first two lock the table their query reads and were observed RED first, with the bare await sitting on the lock until the 10s outer wrap fired. The marker pair needed a test-only seam rather than a lock. Both queries read tables that earlier queries in the same request touch first, so a LOCK TABLE aimed at them stalls an earlier clamp instead, and budget exhaustion has the same problem because an earlier query elapses before the pair is reached. The remaining window between pin_sources_for_oid returning and the pair running is a single get_repo_by_id round trip, measured at 0.14 to 0.17 ms, which is too small to time a lock into without flaking. So there is a cfg(test) stall armed per target, placed INSIDE the timeout wrapper so the clamp is what fires. Every seam item is cfg(test), including the function itself, so a non-test caller would not compile, and a full release build is clean. Each of the four asserts the same contract: a 503 inside the budget, the budget taint in the body, the walk permit returned, and a follow-up request admitted rather than capacity-shed. All four clamps are proven load-bearing by reverting the clamp alone with an attributed message, and the mutations cross-isolate: reverting one turns only its own test red, so each test stalls at its own query rather than a sibling's. --- crates/gitlawb-node/src/api/ipfs.rs | 495 ++++++++++++++++++++++++++-- 1 file changed, 466 insertions(+), 29 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f42a3b61..86d62932 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -261,6 +261,22 @@ pub async fn get_by_cid( // handler's `auth` extension, so resolve it once here. let caller_owned = auth.as_ref().map(|e| e.0 .0.as_str().to_string()); + // Every DB await from here on runs while the scarce walk permits are ALREADY + // held, and the pool sets no statement_timeout, so an unclamped query blocked in + // Postgres would pin those slots for the whole stall, past the request budget, + // and capacity-503 later requests from any unauthenticated caller (#174 F2). + // Each one is clamped to the request deadline; returning on the timeout arm + // RAII-drops `admission`, which is the whole mechanism, so no new state is + // needed. Defined once here so the clamp sites on the provenance path share one + // definition (the legacy-scan preload below keeps its own `budget_shed` inside + // its nested scope). + let budget_shed = || { + AppError::Overloaded(format!( + "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" + )) + }; + let remaining = || request_deadline.saturating_duration_since(std::time::Instant::now()); + // Resolve the content-addressed CID to the object's git oid(s). A real pin // CID digests the raw object content (`Cid::from_git_object_bytes`), NOT the // git oid (git frames content with a `" \0"` header first), so we @@ -271,11 +287,19 @@ pub async fn get_by_cid( // when the chosen one is withheld or absent while another is readable (#173). // An empty result is an opaque 404, uniform with a genuine not-found and a // visibility denial. - let oids = state - .db - .oids_for_cid(&canonical_cid) - .await - .map_err(AppError::Internal)?; + let oids = match tokio::time::timeout(remaining(), state.db.oids_for_cid(&canonical_cid)).await + { + Ok(Ok(v)) => v, + Ok(Err(e)) => return Err(AppError::Internal(e)), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs oids_for_cid exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; if oids.is_empty() { return Err(AppError::RepoNotFound(format!( "no git object found for CID {cid_str}" @@ -323,11 +347,23 @@ pub async fn get_by_cid( // scan fan-out. A shared object first pinned from a private/quarantined repo // still serves from a later PUBLIC source. Deterministic (ORDER BY on the // union), so no ordering can turn an authorized copy into a 404. - let sources = state - .db - .pin_sources_for_oid(sha256_hex) - .await - .map_err(AppError::Internal)?; + let sources = match tokio::time::timeout( + remaining(), + state.db.pin_sources_for_oid(sha256_hex), + ) + .await + { + Ok(Ok(v)) => v, + Ok(Err(e)) => return Err(AppError::Internal(e)), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs pin_sources_for_oid exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; // Provenance fast-path: try each recorded source repo through the SAME gate // (bounded to first-pinner + MAX_PIN_SOURCES). Empty for a legacy NULL-provenance // pin. The first source that authorizes serves — no scan fan-out on the common @@ -341,13 +377,6 @@ pub async fn get_by_cid( // here drops the permits. The quarantine bit and the visibility rules are // both access control, so a timeout must DENY rather than fall through with // an empty answer (FAIL CLOSED). - let budget_shed = || { - AppError::Overloaded(format!( - "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" - )) - }; - let remaining = - || request_deadline.saturating_duration_since(std::time::Instant::now()); let repo = match tokio::time::timeout(remaining(), state.db.get_repo_by_id(repo_id)) .await { @@ -473,20 +502,48 @@ pub async fn get_by_cid( continue; } } - let needs_scan = sources.is_empty() || { - #[cfg(test)] - bump_marker_queries(); - state - .db - .pin_sources_at_cap(sha256_hex) + let needs_scan = sources.is_empty() + || { + #[cfg(test)] + bump_marker_queries(); + let at_cap = match tokio::time::timeout(remaining(), async { + #[cfg(test)] + stall_marker_query(MarkerQuery::AtCap).await; + state.db.pin_sources_at_cap(sha256_hex).await + }) .await - .map_err(AppError::Internal)? - || state - .db - .pin_sources_incomplete(sha256_hex) + { + Ok(Ok(v)) => v, + Ok(Err(e)) => return Err(AppError::Internal(e)), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs pin_sources_at_cap exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; + at_cap + || match tokio::time::timeout(remaining(), async { + #[cfg(test)] + stall_marker_query(MarkerQuery::Incomplete).await; + state.db.pin_sources_incomplete(sha256_hex).await + }) .await - .map_err(AppError::Internal)? - }; + { + Ok(Ok(v)) => v, + Ok(Err(e)) => return Err(AppError::Internal(e)), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs pin_sources_incomplete exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(budget_shed()); + } + } + }; if needs_scan { // Load the scan context once, lazily (shared across oid candidates). if scan_ctx.is_none() { @@ -1305,6 +1362,61 @@ fn bump_marker_queries() { MARKER_QUERIES.with(|c| c.set(c.get() + 1)); } +/// Which of the two `needs_scan` marker queries a test wants to stall. +#[cfg(test)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum MarkerQuery { + AtCap, + Incomplete, +} + +// Test-only fault-injection seam for the `needs_scan` marker pair +// (`pin_sources_at_cap`, `pin_sources_incomplete`), same idea as +// `RepoStore::tigris_stall`: hold one specific await open so the clamp around it is +// the one observed to fire. +// +// A `LOCK TABLE` fixture cannot isolate these two. `pin_sources_at_cap` reads +// `pin_repo_sources` and `pin_sources_incomplete` reads `pinned_cids`, and BOTH tables +// are already read by `oids_for_cid` and `pin_sources_for_oid` earlier in the same +// admission-held region, so a lock taken before the request stalls one of those instead +// and the RED is attributed to the wrong clamp. Taking the lock mid-request does not +// help either: the window between `pin_sources_for_oid` returning and this pair running +// is a single `get_repo_by_id` round trip (measured at ~0.15ms against this Postgres), +// so timing a lock into it is a race that flakes under load. +// +// Armed per target so the second query can be reached with the first left untouched +// (`at_cap` must return `false` for the `||` to evaluate `pin_sources_incomplete`). +// `thread_local` for the same reason as the counters above: `#[sqlx::test]` runs each +// case on its own current-thread runtime, so arming here is invisible to cases running +// in parallel. +#[cfg(test)] +thread_local! { + static MARKER_QUERY_STALL: std::cell::Cell> = + const { std::cell::Cell::new(None) }; +} + +#[cfg(test)] +pub(crate) fn arm_marker_query_stall(which: MarkerQuery, stall: std::time::Duration) { + MARKER_QUERY_STALL.with(|c| c.set(Some((which, stall)))); +} + +#[cfg(test)] +pub(crate) fn disarm_marker_query_stall() { + MARKER_QUERY_STALL.with(|c| c.set(None)); +} + +/// Awaited INSIDE each marker query's `tokio::time::timeout`, never before it: a stall +/// placed outside the clamp would elapse with the clamp never firing and prove nothing. +#[cfg(test)] +async fn stall_marker_query(which: MarkerQuery) { + let armed = MARKER_QUERY_STALL.with(|c| c.get()); + if let Some((target, stall)) = armed { + if target == which { + tokio::time::sleep(stall).await; + } + } +} + // Test-only INV-10 cost counter (F6, U6/U7): how many times the serve path withheld an // object because it exceeded `ipfs_max_served_object_bytes`. The bounded read must reject // an oversized object rather than buffer it on the worker; the counter is the both-ways @@ -1342,6 +1454,7 @@ mod tests { //! CID-resolution / visibility-gate behavior of the handler itself is covered by the //! `#[sqlx::test]` suite in `test_support.rs`. + use super::{arm_marker_query_stall, disarm_marker_query_stall, MarkerQuery}; use axum::body::Body; use axum::extract::ConnectInfo; use axum::http::{Method, Request, StatusCode}; @@ -3581,6 +3694,178 @@ mod tests { ); } + /// F2 (#174): `oids_for_cid` is the FIRST DB await inside the admission-held + /// region, and pre-fix it was a bare await with no deadline. A query blocked in + /// Postgres there pinned both walk permits for the whole stall, past the request + /// budget, so later /ipfs requests took capacity 503s long after + /// GITLAWB_IPFS_REQUEST_BUDGET_SECS elapsed, reachable by any unauthenticated + /// caller. Here `pinned_cids` (the only table `oids_for_cid` reads) is held + /// ACCESS EXCLUSIVE so the query blocks at lock acquisition. + /// + /// The follow-up after ROLLBACK is a 404 rather than a 200 because this scenario + /// seeds no pin at all; "admitted and answered, never capacity-503'd" is what + /// proves the permit came back. + /// + /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrapping + /// timeout fires (RED). MUTATION (RED): drop the `tokio::time::timeout` around + /// `oids_for_cid` and this hangs past the wrap. + #[sqlx::test] + async fn get_by_cid_stalled_oids_query_frees_walk_permit(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held/freed permit is directly observable; + // per-source cap permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let sem = state.git_ipfs_walk_semaphore.clone(); + // A well-formed CID with no `pinned_cids` row: the request still runs the + // `oids_for_cid` lookup, which is the await under test. + let cid = cid_for_oid(&absent_oid()); + let router = ipfs_router(state); + + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE pinned_cids IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.clone().oneshot(get_cid(&cid, Some(peer))), + ) + .await + .expect("the budget clamp must return within budget; a bare await hangs on the lock") + .unwrap(); + let elapsed = started.elapsed(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an oid lookup blocked past the request budget must shed a retryable 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~budget (1s); got {elapsed:?} \ + (pre-fix the bare await blocks on the lock for the whole stall)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the shed must name the budget taint so it maps to \ + GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must be freed on the budget-shed path, not held for the stall" + ); + + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + let resp2 = router.oneshot(get_cid(&cid, Some(peer))).await.unwrap(); + assert_eq!( + resp2.status(), + StatusCode::NOT_FOUND, + "with the permit freed and the lock released, a follow-up is ADMITTED and \ + answers 404 (no pin was seeded), never capacity-503'd" + ); + } + + /// F2 (#174), second lockable site: `pin_sources_for_oid` runs once per candidate + /// oid, still inside the admission-held region, and was likewise a bare await. + /// + /// The lock isolates it from the first await: `oids_for_cid` reads only + /// `pinned_cids`, which stays unlocked, so it completes and the handler reaches + /// the per-oid loop; `pin_sources_for_oid` also reads `pin_repo_sources`, which is + /// held ACCESS EXCLUSIVE, so it is the query that blocks. A seeded legacy pin is + /// what gives the loop an oid to iterate. + /// + /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrap + /// fires (RED). MUTATION (RED): drop the `tokio::time::timeout` around + /// `pin_sources_for_oid`; that mutation must leave the `oids_for_cid` scenario + /// above GREEN, which is what proves the two tests isolate their own queries. + #[sqlx::test] + async fn get_by_cid_stalled_pin_sources_query_frees_walk_permit(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let sem = state.git_ipfs_walk_semaphore.clone(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let router = ipfs_router(state); + + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE pin_repo_sources IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.82:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.clone().oneshot(get_cid(&cid, Some(peer))), + ) + .await + .expect("the budget clamp must return within budget; a bare await hangs on the lock") + .unwrap(); + let elapsed = started.elapsed(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a pin-source lookup blocked past the request budget must shed a retryable 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~budget (1s); got {elapsed:?} \ + (pre-fix the bare await blocks on the lock for the whole stall)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the shed must name the budget taint so it maps to \ + GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must be freed on the budget-shed path, not held for the stall" + ); + + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + let resp2 = router.oneshot(get_cid(&cid, Some(peer))).await.unwrap(); + assert_eq!( + resp2.status(), + StatusCode::NOT_FOUND, + "with the permit freed and the lock released, a follow-up is served (404 \ + against an empty repo set), never capacity-503'd" + ); + } + /// F6/KTD-5 FAIL CLOSED (security-critical): `list_visibility_rules_for_repos` is /// the access-control query. If its timeout let the handler fall through with an /// empty rule map, the loop would apply no visibility rules and serve an unfiltered @@ -3685,4 +3970,156 @@ mod tests { .unwrap(); drop(lock_conn); } + + /// Seed a PROVENANCED pin whose single recorded source repo does not exist, and + /// return its CID. `pin_sources_for_oid` therefore comes back NON-EMPTY (so + /// `needs_scan` cannot short-circuit on `sources.is_empty()`) while the per-source + /// loop takes the `get_repo_by_id -> None` arm and falls straight through to the + /// marker pair. `pin_repo_sources` stays empty, so `pin_sources_at_cap` is `false` + /// and the `||` goes on to evaluate `pin_sources_incomplete`. + async fn seed_provenanced_pin_with_missing_source( + state: &crate::state::AppState, + oid: &str, + ) -> String { + let cid = cid_for_oid(oid); + state + .db + .record_pinned_cid(oid, &cid, Some("repo-id-that-does-not-exist")) + .await + .expect("seed a provenanced pin row"); + cid + } + + /// Shared body for the two marker-query stall cases. Arms the seam for `which`, + /// drives one request, and asserts the whole budget-shed contract: a 503 inside the + /// budget, a body naming the budget taint, the walk permit BACK in the pool rather + /// than pinned for the stall, and a follow-up that is ADMITTED (404 here, since the + /// fixture seeds no servable object) instead of capacity-shed. + async fn assert_marker_query_stall_frees_walk_permit( + pool: sqlx::PgPool, + which: MarkerQuery, + peer: SocketAddr, + label: &str, + ) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held/freed permit is directly observable; + // per-source cap permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let sem = state.git_ipfs_walk_semaphore.clone(); + let cid = seed_provenanced_pin_with_missing_source(&state, &absent_oid()).await; + let router = ipfs_router(state); + + // 30s so the stall is decided by the 1s budget clamp, never by the sleep + // finishing on its own. + arm_marker_query_stall(which, std::time::Duration::from_secs(30)); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.clone().oneshot(get_cid(&cid, Some(peer))), + ) + .await + .unwrap_or_else(|_| { + disarm_marker_query_stall(); + panic!("{label}: the budget clamp must return within budget; a bare await hangs") + }) + .unwrap(); + let elapsed = started.elapsed(); + disarm_marker_query_stall(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "{label}: a marker query blocked past the request budget must shed a retryable 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "{label}: the clamp must end the request at ~budget (1s); got {elapsed:?} \ + (an unclamped await runs the full 30s stall)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "{label}: the shed must name the budget taint so it maps to \ + GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + // The scarce walk permit was RAII-dropped on the early return, not pinned for + // the stall: the slot is free again the instant the request returns. + assert_eq!( + sem.available_permits(), + 1, + "{label}: the walk permit must be freed on the budget-shed path, not held \ + for the stall" + ); + + // With the seam disarmed the follow-up is ADMITTED and answered (404, since the + // fixture seeds no servable object), never capacity-503'd, which is what proves + // the slot came back rather than staying pinned. + let resp2 = router.oneshot(get_cid(&cid, Some(peer))).await.unwrap(); + assert_eq!( + resp2.status(), + StatusCode::NOT_FOUND, + "{label}: with the permit freed, a follow-up is admitted and answers 404, \ + never capacity-503'd" + ); + } + + /// F2 (#174), marker pair, first query: `pin_sources_at_cap` runs on a provenance + /// MISS, still inside the admission-held region, and pre-fix was a bare await. A + /// query blocked in Postgres there pinned the walk permits for the whole stall, + /// past the request budget, capacity-503'ing later requests from any + /// unauthenticated caller. + /// + /// This one needs the `#[cfg(test)]` fault-injection seam rather than a `LOCK TABLE` + /// fixture: it reads `pin_repo_sources`, which `pin_sources_for_oid` already read + /// earlier in the same region, so a table lock stalls that earlier await and the RED + /// lands on the wrong clamp. See `MARKER_QUERY_STALL` for the full reasoning. The + /// injected sleep sits INSIDE the `tokio::time::timeout`, so the clamp is what ends + /// the request. + /// + /// MUTATION (RED): replace the clamp around `pin_sources_at_cap` with the bare + /// await, keeping the seam, and the request runs the full 30s stall past the 10s + /// wrap. + #[sqlx::test] + async fn get_by_cid_stalled_pin_sources_at_cap_frees_walk_permit(pool: sqlx::PgPool) { + assert_marker_query_stall_frees_walk_permit( + pool, + MarkerQuery::AtCap, + "203.0.113.83:5000".parse().unwrap(), + "pin_sources_at_cap", + ) + .await; + } + + /// F2 (#174), marker pair, second query: `pin_sources_incomplete` is a SEPARATE + /// clamp, evaluated only when `pin_sources_at_cap` came back `false`, and carries + /// the same pre-fix bare-await exposure. The fixture leaves `pin_repo_sources` + /// empty so `at_cap` is `false` and the `||` actually reaches this query; the seam + /// is armed for `Incomplete` only, so the first query is untouched and the RED is + /// attributable to this clamp alone. + /// + /// It reads `pinned_cids`, which `oids_for_cid` and `pin_sources_for_oid` already + /// read, so it is unlockable for the same reason as its sibling above. + /// + /// MUTATION (RED): replace the clamp around `pin_sources_incomplete` with the bare + /// await, keeping the seam, and the request runs the full 30s stall past the 10s + /// wrap. + #[sqlx::test] + async fn get_by_cid_stalled_pin_sources_incomplete_frees_walk_permit(pool: sqlx::PgPool) { + assert_marker_query_stall_frees_walk_permit( + pool, + MarkerQuery::Incomplete, + "203.0.113.84:5000".parse().unwrap(), + "pin_sources_incomplete", + ) + .await; + } } From 9984e082121f3e8d25750dd54c0d60c85cef0ac7 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 10 Aug 2026 20:36:04 -0500 Subject: [PATCH 35/77] fix(node): bound the pin loops' DB work by the batch deadline, both lanes api/repos.rs holds the global pin_semaphore across ipfs_pin::pin_new_objects and across pinata::pin_new_objects. Inside both, batch_budget_gate only gates between objects and the git read was already clamped, but the DB calls were bare awaits. One stalled query parked the permit past every budget, and once all pin permits were so held, post-push IPFS and Pinata replication stopped for every repository on the node. The pool defers rather than sheds, so the queue behind it only grows. The bound set is derived callee-transitively from each budgeted region's call graph, not from the shape of the fix. A grep for db./retry_db_record over the loop bodies cannot see a DB await behind a call site, and that is not hypothetical: it missed repair_legacy_provider_cid, whose own cid_for_oid and repair_legacy_provider_cid awaits run under the permit from both lanes while its deadline argument bounds only the spawn_blocking git read. Kubo: six direct sites plus those two transitive ones. Pinata: eight direct sites, two more than Kubo because its post-upload path has its own record_pin_source and incomplete marker. Completeness here is the enumeration, scoped to the call graph at this head, not a compile-enforced guarantee. db_bounded takes the ABSOLUTE deadline, so a slow predecessor cannot hand a later call a fresh budget. What its Elapsed arm MEANS is a property of the operation, not of the timeout. A multi-statement transaction whose tx.commit() is never reached definitively did not land: record_pin_source is pool.begin(), INSERT, conditional UPDATE, tx.commit(), so a future cancelled while the INSERT waits on a lock cannot have committed. A single autocommit statement may well have landed server-side after the client future was dropped, since tokio cancels the future but not the statement Postgres is running; mark_pin_sources_incomplete and record_pinata_cid are that shape. Each site maps its elapsed arm on that basis. Reads skip the object. Provenance and backfill warn and continue, since the backfill is idempotent under its repo_id IS NULL guard. A timed-out record_pin_source writes the incomplete marker, exactly as its definite-error arm does, because the source definitively was not recorded and leaving the set incomplete AND unmarked is the state this change exists to prevent: /ipfs would read a non-empty below-cap source set as complete and 404 a copy the repo would serve. The arms stay separate so the warn still distinguishes a stalled batch from scattered per-object failures. The durability writes take a floored remainder. batch_budget_gate only guarantees PIN_READ_FLOOR before an object starts and the add gets the whole remainder, so a successful add can finish with ~0 budget left; an unfloored bound would fail a write that today completes in milliseconds. That leaves bytes in Kubo with no pinned_cids row, and on the Pinata side it is worse, because pinned.push is unconditional there, so repos.rs would build cid_map and gossip publish_ref_update for a CID that /ipfs will 404. The floor is re-taken from now at each call, so graces chain within one iteration: the worst case is deadline+4s on the Kubo skip branch and deadline+6s on the Pinata add path, about 5% of a 120s PIN_BATCH_BUDGET. It does not stack per object, because batch_budget_gate breaks at the next iteration's first statement. pin_new_objects_gated now takes the batch budget instead of hardcoding PIN_BATCH_BUDGET. The permit-release regression asserts a bound that IS the budget, so with the constant inlined it cost 120s of wall clock; passing it drops the test to 3.4s. Production passes the same constant. Both twin doc comments are updated together. Each previously stated that the DB round-trips were untimed inside the budgeted region, which is now false. The Pinata permit's total hold across pinata_object_list_for_refs stays unbounded and is called out as such: repos.rs acquires the permit before that pre-loop walk, so this change closes the DB-stall trigger, not the git-walk one. Tests: thirteen regressions, the stall cases observed RED first with the bare await sitting on the lock until the outer wrap fired. Every guard is proven load-bearing by a mutation carrying an attributed message, so a RED is pinned to the named property rather than to seam removal. Three properties needed a fixture built specifically to bind them, because the obvious test passed either way. The record floor: a zero remainder still landed the write, since tokio polls the inner future before the timer and a local UPDATE round-trips inside one tick, so the marker's table is now locked and released inside the grace window, which fails without the floor and lands with it. The absolute deadline: every single-object stall test passes under a per-call duration too, so a helper unit test now runs two calls against one shared deadline and asserts the second is cut off by the remainder its predecessor left, with the mutation granting a fresh duration equal to the budget rather than larger than it. The Pinata post-upload elapsed arm: it had been changed in lockstep with the two covered sites but nothing executed it, so a fixture now lets the upload succeed and stalls only the record that follows. --- crates/gitlawb-node/src/api/repos.rs | 100 ++- crates/gitlawb-node/src/ipfs_pin.rs | 893 ++++++++++++++++++++++++++- crates/gitlawb-node/src/pinata.rs | 603 +++++++++++++++++- 3 files changed, 1544 insertions(+), 52 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index ffed6257..5608f893 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1359,10 +1359,16 @@ async fn pinata_object_list_for_refs( /// retained list memory is not bounded by this pool. Bounding that is a real change to /// the capture shape and is deliberately not attempted here; the Pinata twin below /// avoids it by acquiring BEFORE it derives its list. -// Eight because the merge of #173 and #174 landed both sets of arguments on one +// Nine because the merge of #173 and #174 landed both sets of arguments on one // signature: #174's pin-admission permit plus #173's git seam and pin provenance. // Each is a distinct value the pin loop needs and none is derivable from another, so -// a wrapper struct here would only rename the same eight fields. +// a wrapper struct here would only rename the same nine fields. +// +// `batch_budget` is the ninth and is passed rather than read from the constant so the +// permit-release regression can drive a short budget. Hardcoding `PIN_BATCH_BUDGET` +// here made that test cost 120s of wall clock, and the only cheaper shapes were +// vacuous: a lock released before the budget returns at the same time whether or not +// the loop is bounded, which proves nothing. Production passes the constant. #[allow(clippy::too_many_arguments)] async fn pin_new_objects_gated( pin_sem: &Arc, @@ -1373,6 +1379,7 @@ async fn pin_new_objects_gated( object_list: Vec, db: &Arc, repo_id: &str, + batch_budget: std::time::Duration, ) -> Vec<(String, String)> { // Nothing to pin: answer before taking a permit (#174 F2b). The permit bounds how // many pin loops run concurrently, and an empty list does no pinning, so parking @@ -1395,7 +1402,7 @@ async fn pin_new_objects_gated( object_list, db, repo_id, - crate::ipfs_pin::PIN_BATCH_BUDGET, + batch_budget, ) .await } @@ -1427,6 +1434,7 @@ async fn pin_and_encrypt_objects( // The drain's repo id, never `ctx.repo_id` frozen at spawn (#174 U3): pin // provenance must name the row the reader will resolve against. repo_id, + crate::ipfs_pin::PIN_BATCH_BUDGET, ) .await; if !pinned.is_empty() { @@ -6897,6 +6905,7 @@ mod tests { objects.clone(), &db, "repo-gated-a", + crate::ipfs_pin::PIN_BATCH_BUDGET, ), ) .await; @@ -6918,6 +6927,7 @@ mod tests { objects, &db, "repo-gated-b", + crate::ipfs_pin::PIN_BATCH_BUDGET, ), ) .await @@ -6925,6 +6935,89 @@ mod tests { assert!(out.is_empty(), "an empty ipfs_api pins nothing"); } + /// #173 F3, at the layer that actually owns the permit. `pin_new_objects_gated` + /// holds the global `pin_semaphore` across the whole `ipfs_pin::pin_new_objects` + /// call, so a DB call inside that loop with no deadline parked a global pin slot + /// for as long as the query was stuck; once every slot was so held, post-push + /// replication stopped for every repo on the node. With the loop's DB calls + /// bounded by the batch deadline the call returns at ~the batch budget and the + /// permit comes back, even though the table is still locked. + /// + /// The endpoint is a LIVE mockito server, not the `""` the sibling test above + /// uses. `ipfs_pin::pin_new_objects` returns `vec![]` immediately on an empty + /// `ipfs_api`, so an empty-string copy would never reach `is_pinned`, never touch + /// the locked table, and pass identically with the bound deleted. The mock is at + /// `.expect(0)` because a stalled pinned-status check must not fall through to an + /// add. + /// + /// The budget is passed in (1500ms) rather than read from `PIN_BATCH_BUDGET`. + /// Before that seam existed this test cost 120s of wall clock, because the bound + /// it asserts IS the batch budget and the gate hardcoded the production constant. + /// 1500ms is deliberately above `PIN_READ_FLOOR` (1100ms): below the floor + /// `batch_budget_gate` breaks the loop as its first statement, so the run would + /// never reach a DB call and would pass with the bound deleted. + #[sqlx::test] + async fn pin_new_objects_gated_frees_permit_after_stalled_db(pool: sqlx::PgPool) { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let state = crate::test_support::test_state(pool.clone()).await; + let db = state.db.clone(); + let tmp = tempfile::TempDir::new().unwrap(); + let pin_sem = Arc::new(Semaphore::new(1)); + + let mut server = mockito::Server::new_async().await; + let add = server + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .with_body(r#"{"Hash":"QmShouldNotHappen"}"#) + .expect(0) + .create_async() + .await; + + // `ACCESS EXCLUSIVE` conflicts with the `ACCESS SHARE` every SELECT needs, so + // `is_pinned` blocks at lock acquisition regardless of row count. Held for the + // whole call: a lock released early would let the pre-fix bare await finish too, + // and the test would prove nothing. + let mut lock = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE pinned_cids IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock) + .await + .unwrap(); + + let objects = vec!["0123456789abcdef0123456789abcdef01234567".to_string()]; + let out = tokio::time::timeout( + std::time::Duration::from_secs(150), + pin_new_objects_gated( + &pin_sem, + &server.url(), + tmp.path(), + "git", + std::time::Duration::from_secs(30), + objects, + &db, + "repo-gated-stalled-db", + std::time::Duration::from_millis(1500), + ), + ) + .await + .expect( + "a stalled DB must cost the pin loop its batch budget, not the lock's \ + lifetime: an unbounded await inside the loop holds this permit past every \ + budget", + ); + + assert!(out.is_empty(), "a stalled pinned-status check pins nothing"); + assert_eq!( + pin_sem.available_permits(), + 1, + "the global pin permit must be back once the bounded loop returns" + ); + add.assert_async().await; + + sqlx::raw_sql("ROLLBACK").execute(&mut *lock).await.unwrap(); + } + /// #174 F2b: the pin permit bounds how many pin loops run concurrently, so a call /// with NOTHING to pin must not take one. It otherwise spends a global pin slot on no /// work, and the pool DEFERS rather than sheds, so those calls stall pins for every @@ -6956,6 +7049,7 @@ mod tests { vec![], &db, "repo-gated-empty", + crate::ipfs_pin::PIN_BATCH_BUDGET, ), ) .await diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 62621b98..cd704a33 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -47,6 +47,110 @@ where } } +/// The smallest bound a durability write is given, however little of the batch +/// deadline is left. +/// +/// `batch_budget_gate` only guarantees [`PIN_READ_FLOOR`] before an object STARTS, +/// and the add is handed the whole remainder, so a successful add can finish with +/// ~0 left. An unfloored bound would then fail a write that today completes in +/// milliseconds: on the add path the bytes would sit in Kubo with no `pinned_cids` +/// row, so nothing could resolve the CID, and on the skip branch the source record +/// would fail AND its compensating `mark_pin_sources_incomplete` would fail with +/// it, producing exactly the incomplete-set-without-marker state the marker exists +/// to prevent. The grace exists so a spent batch deadline degrades to a slightly +/// late permit release, never to a dropped durability write. +pub(crate) const DB_RECORD_GRACE: Duration = Duration::from_secs(2); + +/// Why a DB call bounded by the batch deadline did not return a value. +/// +/// The two arms are kept apart because an operator has to be able to tell a stalled +/// batch (every object timing out at once) from scattered per-object DB failures, and +/// because what an elapsed bound MEANS is not the same claim as a definite error even +/// where the two lead to the same compensation. Every warn line at a bounded site +/// names which arm fired. +#[derive(Debug)] +pub(crate) enum BoundedDbError { + /// The batch deadline was reached with the call still in flight. + /// + /// Whether this means "definitely did not happen" or "outcome unknown" is a + /// property of the OPERATION, not of the timeout, so each call site has to decide + /// it from the shape of the call it wrapped. `tokio::time::timeout` cancels the + /// client future; it does not cancel a statement Postgres has already started. + /// The two shapes that follow from that: + /// + /// - a MULTI-STATEMENT operation that ends in an explicit `tx.commit()` + /// DEFINITELY did not land. The cancelled future never reaches the commit, so no + /// COMMIT is ever sent and Postgres discards the transaction when the connection + /// is reset. `Db::record_pin_source` and `Db::record_pinned_cid_with_source` are + /// this shape, and a site that compensates for a definite error must compensate + /// here too; + /// - a SINGLE AUTOCOMMIT statement may still land server-side after this arm is + /// taken, because the statement is already running and nothing cancels it. + /// `Db::mark_pin_sources_incomplete` and `Db::record_pinata_cid` are this shape, + /// and nothing downstream may treat this arm as evidence the write did not + /// happen. + Elapsed, + /// The DB operation itself failed, definitely and with a cause. + Db(anyhow::Error), +} + +impl std::fmt::Display for BoundedDbError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Elapsed => write!(f, "batch deadline reached with the DB call in flight"), + Self::Db(e) => write!(f, "{e}"), + } + } +} + +impl From for anyhow::Error { + fn from(e: BoundedDbError) -> Self { + match e { + BoundedDbError::Elapsed => anyhow::anyhow!("{e}"), + // Keep the real cause chain rather than flattening it to a string. + BoundedDbError::Db(inner) => inner, + } + } +} + +/// Bound one DB operation by the batch deadline. +/// +/// What this bounds is the PERMIT HOLD. Both pin loops run under a global +/// `pin_semaphore` permit and that pool defers rather than sheds, so a bare DB +/// await inside the budgeted region parks the permit for as long as the query is +/// stuck; once every pin permit is so held, post-push IPFS replication stops for +/// every repository on the node. `batch_budget_gate` cannot fix that, because it +/// only gates BETWEEN objects and cannot preempt a call already in flight. +/// +/// Takes the ABSOLUTE `deadline`, not a duration, so a slow predecessor cannot hand +/// a later call a fresh full budget: the remainder is measured from the same fixed +/// point every time, which is what keeps N calls inside ONE budget instead of N. +/// +/// Callers must map the elapsed arm PER SITE, from the shape of the operation they +/// wrapped, rather than folding it into their existing error arm or assuming one +/// meaning for all of them. `timeout` cancels the client future, never the statement +/// Postgres is already running, so an autocommit statement can land server-side after +/// this returns [`BoundedDbError::Elapsed`] while a multi-statement transaction whose +/// `tx.commit()` is never reached definitely cannot. See the arm's own docs for which +/// operations here are which. +pub(crate) async fn db_bounded(deadline: Instant, fut: F) -> Result +where + F: std::future::Future>, +{ + let left = deadline.saturating_duration_since(Instant::now()); + match tokio::time::timeout(left, fut).await { + Ok(Ok(v)) => Ok(v), + Ok(Err(e)) => Err(BoundedDbError::Db(e)), + Err(_elapsed) => Err(BoundedDbError::Elapsed), + } +} + +/// The deadline a durability write gets: the batch deadline, floored at +/// [`DB_RECORD_GRACE`] from now so a spent budget cannot drop the write. +pub(crate) fn db_record_deadline(deadline: Instant) -> Instant { + std::cmp::max(deadline, Instant::now() + DB_RECORD_GRACE) +} + /// Opportunistically repair a legacy provider-CID row on the already-pinned skip /// path (#173 R8, KTD8). Releases before this branch stored the PROVIDER CID /// (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`; the `/ipfs` resolver @@ -75,7 +179,11 @@ pub(crate) async fn repair_legacy_provider_cid( sha: &str, db: &crate::db::Db, ) -> Result { - let stored = match db.cid_for_oid(sha).await? { + // Bounded by the SAME `deadline` the git read below uses (F3, #173): both pin + // loops call this with the pin permit held, so a bare await here parked that + // permit exactly the way the loop bodies' own awaits did. A grep over the loop + // bodies cannot see this site, which is why it is bounded from inside. + let stored = match db_bounded(deadline, db.cid_for_oid(sha)).await? { Some(c) => c, None => return Ok(RepairOutcome::Settled), }; @@ -130,7 +238,7 @@ pub(crate) async fn repair_legacy_provider_cid( if raw == stored { return Ok(RepairOutcome::Settled); } - db.repair_legacy_provider_cid(sha, &raw, &stored).await?; + db_bounded(deadline, db.repair_legacy_provider_cid(sha, &raw, &stored)).await?; Ok(RepairOutcome::Repaired) } @@ -591,7 +699,7 @@ pub(crate) fn batch_budget_gate( /// /// The loop holds a `pin_semaphore` permit and that pool defers rather than /// sheds, so the hold has to be bounded by something other than the pusher's -/// object count. Three things here are: +/// object count. Four things here are: /// /// - this loop's own wall-clock: the deadline is taken once at loop start and /// checked at the top of every iteration, so no object's work begins with less @@ -607,12 +715,32 @@ pub(crate) fn batch_budget_gate( /// as its per-request timeout, which is what lets one large healthy upload run past /// the shared client's 10s default without letting the batch run forever. Measuring /// it after the read is what keeps the read-plus-add pair inside one budget rather -/// than up to two of them. +/// than up to two of them; +/// - the DB round-trips: every DB operation reachable from inside the region is +/// bounded by the same absolute deadline through [`db_bounded`], including the two +/// inside `repair_legacy_provider_cid`, which the loop body's own call sites do not +/// show. `retry_db_record` is wrapped as a whole so its ladder cannot multiply one +/// remainder, and the durability writes (the post-add record, the skip branch's +/// source record and its incomplete marker) take the floored remainder +/// `max(remaining, DB_RECORD_GRACE)` so a spent budget delays the permit release +/// rather than dropping a write. A bound is not a rollback, and what an elapsed +/// bound MEANS is a property of the operation, so each site maps that arm from the +/// shape of the call it wrapped: a multi-statement transaction whose `tx.commit()` +/// is never reached definitely did not land and is compensated like a definite +/// error, while a single autocommit statement may still land server-side and is +/// never treated as a failed write. See [`BoundedDbError::Elapsed`]. /// -/// So the LOOP's hold is bounded by roughly `batch_budget` plus one teardown. Two -/// things inside that region still are not, and the gate cannot fix either: +/// So the LOOP's hold is bounded by roughly `batch_budget` plus one teardown plus the +/// record graces one iteration can chain. `db_record_deadline` re-floors from +/// `Instant::now()` at EVERY call, so the graces inside a single iteration add up +/// rather than sharing one floor: this loop's worst case is the skip branch at +/// `deadline + 4s` (the source record, then its incomplete marker). It does NOT stack +/// per object, because the next iteration's first statement is `batch_budget_gate`, +/// which breaks the batch, so the overrun is one iteration's worth however many +/// objects the push carried. Against the 120s `PIN_BATCH_BUDGET` that is roughly a 5% +/// overrun for the batch, not an unbounded hold. One thing inside that region still is +/// not bounded at all, and the gate cannot fix it: /// -/// - the DB round-trips (`is_pinned`, `record_pinned_cid`). /// - the pool. `api::repos` acquires the same `pin_semaphore` for the Pinata /// replication task and holds it across `pinata_object_list_for_refs`, a full git /// re-derivation that runs BEFORE `pinata::pin_new_objects` is entered and whose @@ -681,11 +809,22 @@ pub async fn pin_new_objects( // would never resolve to one repo and known CIDs keep hitting the scan. The // backfill only sets repo_id (AND repo_id IS NULL guard preserves // first-pinner-owns) and never re-pins the bytes: the object is already on IPFS. - match db.is_pinned(&sha).await { + // Every DB call from here to the end of the iteration is bounded by the + // ABSOLUTE batch deadline (F3, #173): the loop runs under a global pin permit + // and a bare await parked it for the whole stall. The elapsed arm is mapped per + // site below, never as a blanket "existing error arm": a timeout cancels the + // client future but not the statement Postgres is running, so it reports an + // UNKNOWN outcome, not a failed write. + match db_bounded(deadline, db.is_pinned(&sha)).await { Ok(true) => { - match db.provenance_for_oid(&sha).await { + // Elapsed here is free to skip: these are reads, so a late server-side + // completion costs nothing, and the backfill's own `AND repo_id IS NULL` + // guard makes a late-landing write idempotent. + match db_bounded(deadline, db.provenance_for_oid(&sha)).await { Ok(None) => { - if let Err(e) = db.backfill_pin_provenance(&sha, repo_id).await { + if let Err(e) = + db_bounded(deadline, db.backfill_pin_provenance(&sha, repo_id)).await + { tracing::warn!(sha = %sha, err = %e, "failed to backfill pin provenance"); } } @@ -700,15 +839,67 @@ pub async fn pin_new_objects( // and without it `GET /ipfs/{cid}` only ever knows the first pinner, so a // shared object first pinned from a private/quarantined repo 404s even // when this repo would serve it. Bounded per object (MAX_PIN_SOURCES). - if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { - tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); - // U3 (#173): the retries are spent and this repo is NOT in the source - // set, so the set is known incomplete. Persist that, or the resolver - // reads a non-empty below-cap set as COMPLETE and 404s an object this - // repo would serve. Warn-only in turn: if the marker write also fails - // the object degrades to the pre-U3 behavior, never worse. - if let Err(e) = db.mark_pin_sources_incomplete(&sha).await { - tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + // The retry ladder is wrapped AS A WHOLE, not per attempt: three stalls + // plus their backoff otherwise multiply one remainder by three. Floored + // at DB_RECORD_GRACE because this is a durability write. + match db_bounded( + db_record_deadline(deadline), + retry_db_record(|| db.record_pin_source(&sha, repo_id)), + ) + .await + { + Ok(()) => {} + // Elapsed here is a DEFINITE non-write, not an unknown outcome, and + // that follows from what was wrapped rather than from the timeout. + // `record_pin_source` is an explicit transaction (`pool.begin()`, + // the insert, a conditional marker clear, `tx.commit()`), so a + // cancelled future never reaches the commit, no COMMIT is ever sent, + // and the row cannot have landed. The source set is therefore + // incomplete and must be marked, exactly as on the definite-error + // arm below; leaving it unmarked is the state the marker exists to + // prevent, since the resolver reads a non-empty below-cap set as + // COMPLETE and 404s a copy this repo would serve. The cost of the + // marker is bounded: the fallback legacy scan is capped at + // `ipfs_max_legacy_probes` and charges the per-IP work rate limiter + // per probe. The arm stays separate only so the warn tells an + // operator a stalled batch from a scattered per-object failure. + Err(e @ BoundedDbError::Elapsed) => { + tracing::warn!( + sha = %sha, + err = %e, + "pin source record did not complete inside the batch deadline; \ + a cancelled multi-statement transaction never commits, so the \ + source is definitely missing and the set is marked incomplete" + ); + if let Err(e) = db_bounded( + db_record_deadline(deadline), + db.mark_pin_sources_incomplete(&sha), + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } + } + // U3 (#173): the retries are spent on REAL errors, so this repo is + // definitely NOT in the source set and the set is known incomplete. + // Persist that, or the resolver reads a non-empty below-cap set as + // COMPLETE and 404s an object this repo would serve. Warn-only in + // turn: if the marker write also fails the object degrades to the + // pre-U3 behavior, never worse. Floored for the same reason the + // record above is: a spent budget must not drop the compensation. + // The marker write itself is a single autocommit statement, so ITS + // own elapsed arm genuinely is an unknown outcome; nothing branches + // on it, which is why warn-only is the right handling there. + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + if let Err(e) = db_bounded( + db_record_deadline(deadline), + db.mark_pin_sources_incomplete(&sha), + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } } } // R8 (#173 round 10): opportunistically repair a legacy provider-CID @@ -867,8 +1058,24 @@ pub async fn pin_new_objects( // sha-to-cid `cid_map` from it, which drives `upsert_branch_cid` and the // p2p `publish_ref_update` gossip CID. Do not re-align them without moving // that consumer first. - match retry_db_record(|| db.record_pinned_cid_with_source(&sha, &raw_cid, repo_id)) - .await + // + // The bound here is FLOORED at DB_RECORD_GRACE. The add was handed the + // whole remainder, so a successful one can return with ~0 left, and an + // unfloored bound would fail a write that today completes in + // milliseconds, leaving the bytes in Kubo with no row to resolve them + // by. If it still fires, both arms mean the same thing here and the site + // takes one Err path for them: `record_pinned_cid_with_source` is an + // explicit transaction, so a cancelled future never reaches its + // `tx.commit()` and the rows definitely did not land, exactly as on a + // real error. Either way the pin is not returned and the next push + // re-offers the object. The warn still names the arm through the error's + // own Display, so an operator can tell a stalled batch from a scattered + // per-object failure. + match db_bounded( + db_record_deadline(deadline), + retry_db_record(|| db.record_pinned_cid_with_source(&sha, &raw_cid, repo_id)), + ) + .await { Ok(()) => pinned.push((sha, cid)), Err(e) => { @@ -1764,4 +1971,648 @@ mod tests { "one corrupt object must cost only itself: the other four must still pin" ); } + + // --------------------------------------------------------------------- + // F3 (#173, jatmn): the DB operations inside the budgeted region. + // + // `api/repos.rs` holds the GLOBAL `pin_semaphore` permit across the whole + // `pin_new_objects` call. `batch_budget_gate` only gates BETWEEN objects and + // the git read is already clamped, but every DB call in the region used to be + // a bare await, so one stalled query parked the permit past every budget and, + // once all pin permits were so held, post-push IPFS replication stopped for + // every repo on the node. The tests below drive that stall with a + // `LOCK TABLE .. IN ACCESS EXCLUSIVE MODE` held on a dedicated pooled + // connection, the same technique as `get_by_cid_stalled_metadata_query_frees_ + // walk_permit` in api/ipfs.rs, and copy its tolerances (a ~1s budget, an + // `elapsed < 3s` assertion, a 10s outer wrap). Pre-fix each one blocks on the + // lock until the outer wrap fires. + // --------------------------------------------------------------------- + + /// Take an `ACCESS EXCLUSIVE` lock on `table` on a dedicated pooled connection. + /// Every SELECT needs `ACCESS SHARE`, which conflicts, so the next statement + /// touching the table blocks at lock acquisition regardless of row count. + async fn lock_table( + pool: &sqlx::PgPool, + table: &str, + ) -> sqlx::pool::PoolConnection { + let mut conn = pool.acquire().await.unwrap(); + sqlx::raw_sql(&format!( + "BEGIN; LOCK TABLE {table} IN ACCESS EXCLUSIVE MODE;" + )) + .execute(&mut *conn) + .await + .unwrap(); + conn + } + + async fn rollback(conn: &mut sqlx::pool::PoolConnection) { + sqlx::raw_sql("ROLLBACK") + .execute(&mut **conn) + .await + .unwrap(); + } + + /// A raw CIDv1 to seed a pin with, so the opportunistic legacy repair takes its + /// cost gate and reads no bytes. The value only has to be a canonical raw key. + fn seed_cid() -> String { + Cid::from_git_object_bytes(b"pin loop seed").to_string() + } + + /// The helper's zero-remainder path, where the absolute deadline is the whole + /// point: a spent deadline must error immediately rather than hand the call a + /// fresh budget. + /// + /// This is a unit test rather than a loop-driven one on purpose. Driving a ~0 + /// `batch_budget` through `pin_new_objects` is VACUOUS: `batch_budget_gate` + /// returns None below [`PIN_READ_FLOOR`] as the first statement of the loop body, + /// so the batch breaks before any DB call and the test passes identically with + /// `db_bounded` deleted. The helper is tested where the zero-remainder path + /// actually runs. + #[tokio::test] + async fn db_bounded_elapsed_deadline_errors_promptly() { + let spent = Instant::now() - Duration::from_secs(5); + let started = std::time::Instant::now(); + let out: Result = db_bounded(spent, async { + tokio::time::sleep(Duration::from_secs(30)).await; + Ok(7u8) + }) + .await; + + assert!( + matches!(out, Err(BoundedDbError::Elapsed)), + "a spent deadline must yield the DISTINGUISHABLE timeout arm, not a value \ + and not a generic DB error: {out:?}" + ); + assert!( + started.elapsed() < Duration::from_secs(1), + "a spent deadline must error at once, not after a fresh full budget; got {:?}", + started.elapsed() + ); + } + + /// The ABSOLUTE half of the same helper, which no loop-level test actually binds. + /// + /// `db_bounded` takes an `Instant`, not a `Duration`, so every call sharing one + /// deadline shares ONE budget: whatever an earlier call spends, a later one no + /// longer has. `pin_new_objects_multi_object_stall_charges_one_budget` covers that + /// end to end, but it only goes red under a per-call duration LARGER than the batch + /// budget (its mutation grants `PIN_BATCH_BUDGET`, 120s, against a 1.5s budget). A + /// defect that handed every call a fresh duration at or below the budget would slip + /// straight past it, so the property is bound here instead, where it lives and where + /// no lock, endpoint, or budget gate stands between the assertion and the helper. + /// + /// Two calls against one 3s deadline: the first spends 2s and succeeds, so the + /// second sees ~1s left and must elapse even though its own work needs only 2s. + /// Any fresh per-call duration of 2s or more, INCLUDING one exactly equal to the 3s + /// budget, would return a value there instead. + #[tokio::test] + async fn db_bounded_shares_one_budget_across_sequential_calls() { + let deadline = Instant::now() + Duration::from_secs(3); + + let first: Result = db_bounded(deadline, async { + tokio::time::sleep(Duration::from_secs(2)).await; + Ok(1u8) + }) + .await; + assert!( + matches!(first, Ok(1)), + "the first call fits well inside the shared budget and must return its \ + value: {first:?}" + ); + + let started = std::time::Instant::now(); + let second: Result = db_bounded(deadline, async { + tokio::time::sleep(Duration::from_secs(2)).await; + Ok(2u8) + }) + .await; + + assert!( + matches!(second, Err(BoundedDbError::Elapsed)), + "a SHARED deadline is consumed by the calls before it: with ~1s of the 3s \ + left, a 2s call must elapse. A fresh per-call DURATION would let it \ + succeed even when that duration is exactly the budget, and N calls would \ + then charge N budgets instead of one: {second:?}" + ); + assert!( + started.elapsed() < Duration::from_millis(1500), + "the second call must be cut off by the REMAINDER (~1s) rather than run its \ + full 2s; got {:?}", + started.elapsed() + ); + } + + /// Scenario 1: the FIRST DB call in the region (`is_pinned`) stalls. With the + /// batch deadline bounding it the loop abandons the object, the budget gate + /// then breaks the batch, and the call returns at ~budget with nothing pinned. + /// Pre-fix the bare await blocks on the lock for the lock's whole lifetime, + /// holding the caller's global pin permit with it. + #[sqlx::test] + async fn pin_new_objects_stalled_db_returns_by_budget(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("stalled.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let endpoint = delaying_endpoint(vec![Duration::ZERO]).await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let mut lock = lock_table(&pool, "pinned_cids").await; + + let started = std::time::Instant::now(); + let pinned = tokio::time::timeout( + Duration::from_secs(10), + pin_new_objects( + &endpoint, + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-stalled-db", + Duration::from_millis(1500), + ), + ) + .await + .expect( + "a stalled DB must cost the batch its budget, not the lock's lifetime: the \ + bare await hangs past this wrap", + ); + let elapsed = started.elapsed(); + + assert!( + pinned.is_empty(), + "a stalled pinned-status check cannot produce a pinned object: {pinned:?}" + ); + assert!( + elapsed < Duration::from_secs(3), + "the batch deadline must end the call at ~budget (1.5s); got {elapsed:?}" + ); + + rollback(&mut lock).await; + } + + /// Scenario 2, and the sharpest unknown-outcome must-not. The object is already + /// pinned, so the loop takes the skip branch and tries to record this repo as an + /// additional source; `pin_repo_sources` is locked, so that insert stalls inside + /// `retry_db_record`. Two properties: + /// + /// - the whole retry ladder (three attempts plus backoff) lives inside ONE + /// remainder, so the call still returns promptly; + /// - on the TIMEOUT arm the incomplete marker is NOT written. A cancelled client + /// future does not cancel the statement Postgres is running, so the source may + /// well be recorded; the marker would force every later `/ipfs` request for the + /// object onto the O(repos) legacy scan, from any unauthenticated caller, on the + /// strength of an outcome the code does not know. Only the definite-error arm + /// marks incomplete. + /// + /// The record site carries the durability floor, so the return lands at ~2s + /// rather than at the 1.5s budget; that is the floor working, not a missed bound. + #[sqlx::test] + async fn pin_new_objects_skip_branch_stalled_record_returns_by_budget(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("skip_stalled.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + db.record_pinned_cid_with_source(&sha, &seed_cid(), "repo-seed") + .await + .unwrap(); + let endpoint = delaying_endpoint(vec![Duration::ZERO]).await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let mut lock = lock_table(&pool, "pin_repo_sources").await; + + let started = std::time::Instant::now(); + let pinned = tokio::time::timeout( + Duration::from_secs(10), + pin_new_objects( + &endpoint, + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-skip-stalled", + Duration::from_millis(1500), + ), + ) + .await + .expect( + "the wrapped retry ladder must fit inside one remainder: the bare \ + retry_db_record hangs past this wrap", + ); + let elapsed = started.elapsed(); + + assert!( + pinned.is_empty(), + "an already-pinned object is skipped, never re-pinned: {pinned:?}" + ); + assert!( + elapsed < Duration::from_secs(3), + "the record's floored remainder must end the call promptly; got {elapsed:?}" + ); + + rollback(&mut lock).await; + drop(lock); + + assert!( + db.pin_sources_incomplete(&sha).await.unwrap(), + "a TIMED-OUT `record_pin_source` definitively did not land: it is an explicit \ + multi-statement transaction, and the cancelled future never reaches \ + `tx.commit()`, so no COMMIT is ever sent and the row cannot exist. The set is \ + therefore incomplete, and leaving it UNMARKED is the exact state the marker \ + exists to prevent: the resolver reads a non-empty below-cap set as complete \ + and 404s a copy this repo would serve" + ); + } + + /// Scenario 7: three objects, one budget. Every object's first DB call stalls on + /// the same lock, and the total must stay near ONE budget rather than one per + /// object. This is the only loop-level scenario where an absolute-deadline bound + /// and a per-call duration could differ; every single-object stall test above + /// passes under either. + #[sqlx::test] + async fn pin_new_objects_multi_object_stall_charges_one_budget(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("multi_stalled.git"); + let oids = seed_loose_blobs(&repo_path, 3); + let endpoint = delaying_endpoint(vec![Duration::ZERO]).await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let mut lock = lock_table(&pool, "pinned_cids").await; + + let started = std::time::Instant::now(); + let pinned = tokio::time::timeout( + Duration::from_secs(10), + pin_new_objects( + &endpoint, + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-multi-stalled", + Duration::from_millis(1500), + ), + ) + .await + .expect("three stalled objects must still cost one budget, not three"); + let elapsed = started.elapsed(); + + assert!(pinned.is_empty(), "nothing can pin against a stalled DB"); + assert!( + elapsed < Duration::from_secs(3), + "three stalled objects must charge ONE budget (1.5s), not one each; got {elapsed:?}" + ); + + rollback(&mut lock).await; + } + + /// Scenario 8, Kubo half of the durability floor. `batch_budget_gate` only + /// guarantees `PIN_READ_FLOOR` before an object STARTS and the add is handed the + /// whole remainder, so a successful add can finish with ~0 left. Without the + /// floor the post-add record would then be failed by a spent deadline, leaving + /// bytes in Kubo with no `pinned_cids` row and nothing able to resolve the CID. + /// + /// Fixture: a 2s budget, a 1.7s add, and `pinned_cids` locked from 500ms (well + /// after `is_pinned` has read it, and still well before the add returns) until + /// 2.4s. The record therefore starts at ~1.72s with ~280ms of budget left and + /// needs ~680ms of lock wait to land, which only the `DB_RECORD_GRACE` floor buys + /// it. + /// + /// The lock time is a MARGIN, not a boundary: taking it at 100ms left `is_pinned` + /// racing it on a loaded box, and losing that race makes the read block, time out, + /// and break the batch, which fails on `pinned.len() == 1` for a reason that has + /// nothing to do with the floor. Any time between the `is_pinned` round trip and + /// the add's 1.7s return proves the same thing. + #[sqlx::test] + async fn pin_add_with_spent_budget_still_records_row(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("spent_budget.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + let endpoint = delaying_endpoint(vec![Duration::from_millis(1700)]).await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let lock_pool = pool.clone(); + let locker = async move { + tokio::time::sleep(Duration::from_millis(500)).await; + let mut conn = lock_table(&lock_pool, "pinned_cids").await; + tokio::time::sleep(Duration::from_millis(1900)).await; + rollback(&mut conn).await; + }; + + let pin = tokio::time::timeout( + Duration::from_secs(15), + pin_new_objects( + &endpoint, + &repo_path, + "git", + Duration::from_secs(30), + oids.clone(), + &db, + "repo-spent-budget", + Duration::from_millis(2000), + ), + ); + let (pinned, ()) = tokio::join!(pin, locker); + let pinned = pinned.expect("the floored record must land well inside this wrap"); + + assert!( + db.is_pinned(&sha).await.unwrap(), + "a successful add whose batch deadline is spent must still land its \ + pinned_cids row: without the floor the bytes sit in Kubo with no row and \ + nothing can resolve the CID" + ); + assert_eq!( + pinned.len(), + 1, + "the durably recorded pin must be returned: {pinned:?}" + ); + } + + /// Scenario 8, the other direction of the floor: the skip branch's DEFINITE-error + /// arm with the budget already spent must still write the incomplete marker. + /// Without it the source set is incomplete AND unmarked, which is exactly the + /// state the marker exists to prevent: the resolver reads a non-empty below-cap + /// set as complete and 404s an object this repo would serve. + /// + /// The definite error is a dropped `pin_repo_sources`, not a timeout: the DROP + /// runs inside a transaction that commits at 1.5s, so the insert blocks on that + /// transaction's lock and then fails outright. With a 1.2s budget the retry + /// ladder therefore returns its definite error at ~1.6s, past the deadline, and + /// only the floor lets the marker write run at all. + #[sqlx::test] + async fn pin_skip_branch_definite_error_with_spent_budget_marks_incomplete(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("definite_error.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + db.record_pinned_cid_with_source(&sha, &seed_cid(), "repo-seed") + .await + .unwrap(); + let endpoint = delaying_endpoint(vec![Duration::ZERO]).await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let mut dropper = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; DROP TABLE pin_repo_sources;") + .execute(&mut *dropper) + .await + .unwrap(); + + let commit = async move { + tokio::time::sleep(Duration::from_millis(1500)).await; + sqlx::raw_sql("COMMIT") + .execute(&mut *dropper) + .await + .unwrap(); + }; + + let pin = tokio::time::timeout( + Duration::from_secs(15), + pin_new_objects( + &endpoint, + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-definite-error", + Duration::from_millis(1200), + ), + ); + let (pinned, ()) = tokio::join!(pin, commit); + pinned.expect("a definite DB error resolves inside the record floor, not the wrap"); + + assert!( + db.pin_sources_incomplete(&sha).await.unwrap(), + "the DEFINITE-error arm must still mark the source set incomplete with the \ + batch deadline spent: an incomplete-and-unmarked set is read as complete and \ + 404s an object this repo would serve" + ); + } + + /// The MARKER's own floor, which the two tests above leave unbound. + /// + /// Both of them assert the marker lands with the batch deadline spent, and both + /// pass with the marker's `db_record_deadline` replaced by the bare `deadline`. + /// The reason is timing, not coverage: the remainder there really is ~0, but + /// `tokio::time::timeout` polls the inner future before it checks the timer, and a + /// local Postgres UPDATE against an uncontended table round-trips inside that one + /// poll. So the unfloored write lands anyway and the floor is never load-bearing. + /// + /// This makes the marker write SLOW, so a zero bound cannot smuggle it through. + /// `mark_pin_sources_incomplete` is `UPDATE pinned_cids`, so `pinned_cids` is held + /// under `ACCESS EXCLUSIVE` from 300ms (after `is_pinned` and `provenance_for_oid` + /// have read it, both round trips inside the first few ms) until 3s. + /// + /// The schedule, with a 1.5s budget and `pin_repo_sources` locked for the whole + /// run so the source record stalls: + /// + /// - ~10ms: `record_pin_source` starts and blocks on the sources lock. Its own + /// floored bound is `now + 2s`, so it elapses at ~2.01s; + /// - ~2.01s: the elapsed arm runs the marker write, which blocks on the + /// `pinned_cids` lock. Floored, its bound is ~4.01s; unfloored it is the spent + /// 1.5s batch deadline, so the bound is ~0 and the blocked UPDATE is cancelled at + /// once, leaving no marker; + /// - ~3.0s: the lock lifts, a full second after the write started and a full second + /// before its floored bound expires, so the floored write lands. + /// + /// Both margins are a full second on purpose. The proof only needs the release to + /// fall strictly between zero and `DB_RECORD_GRACE`, so there is no reason to put + /// it near either end and make the test a race. + #[sqlx::test] + async fn pin_skip_branch_marker_write_needs_the_record_floor(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("marker_floor.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + db.record_pinned_cid_with_source(&sha, &seed_cid(), "repo-seed") + .await + .unwrap(); + let endpoint = delaying_endpoint(vec![Duration::ZERO]).await; + // Asserted on here, and installed for the sibling reason too: `tracing` caches + // a callsite's interest globally on first hit, so a hit from a thread with no + // subscriber caches it as never-interested for the whole binary. + let (logs, _log_guard) = capture_logs(); + + let mut sources_lock = lock_table(&pool, "pin_repo_sources").await; + + let lock_pool = pool.clone(); + let controller = async { + tokio::time::sleep(Duration::from_millis(300)).await; + let mut cids_lock = lock_table(&lock_pool, "pinned_cids").await; + tokio::time::sleep(Duration::from_millis(2700)).await; + rollback(&mut cids_lock).await; + }; + + let started = std::time::Instant::now(); + let pin = tokio::time::timeout( + Duration::from_secs(20), + pin_new_objects( + &endpoint, + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-marker-floor", + Duration::from_millis(1500), + ), + ); + let (pinned, ()) = tokio::join!(pin, controller); + let pinned = pinned.expect("the floored marker write must land well inside this wrap"); + let elapsed = started.elapsed(); + + rollback(&mut sources_lock).await; + drop(sources_lock); + + assert!( + pinned.is_empty(), + "an already-pinned object is skipped, never re-pinned: {pinned:?}" + ); + assert!( + logs.text() + .contains("did not complete inside the batch deadline"), + "the fixture only proves anything if the marker was reached from the ELAPSED \ + arm of the source record, not from the definite-error arm: {}", + logs.text() + ); + assert!( + elapsed < Duration::from_secs(8), + "the call must end at one budget plus the one chained record grace the \ + blocked marker write costs (~3s), never at the lock's lifetime; got \ + {elapsed:?}" + ); + assert!( + db.pin_sources_incomplete(&sha).await.unwrap(), + "the marker write must be given DB_RECORD_GRACE, not the spent batch \ + deadline: it starts here with ~0 of the budget left and needs ~1s to get \ + past the lock, so an unfloored bound cancels it and the source set is left \ + incomplete AND unmarked, which the resolver reads as complete and 404s a \ + copy this repo would serve" + ); + } + + /// The TRANSITIVE site. `repair_legacy_provider_cid` runs on the skip branch under + /// the same permit, and its own `deadline` argument used to bound only the + /// `spawn_blocking` git read: the two DB awaits inside it (`cid_for_oid` and the + /// key rewrite) were bare, so a stall there parked the permit exactly the way the + /// loop-body awaits did. A grep over the loop bodies cannot see this site, which + /// is why it is driven here rather than argued. + /// + /// Fixture, ordered so the stall lands on `cid_for_oid` and nothing earlier: + /// `pin_repo_sources` is locked from the start so the skip branch's source record + /// blocks; at 1.5s `pinned_cids` is locked (nothing is reading it by then) and at + /// 1.6s the first lock is released. + /// + /// What the loop actually does with that, since the timing is easy to misread: when + /// the `pin_repo_sources` lock lifts at 1.6s the insert succeeds, and then + /// `record_pin_source`'s follow-up `UPDATE pinned_cids` immediately blocks on the + /// `pinned_cids` lock taken at 1.5s and eats the rest of the budget, elapsing at + /// 2.2s. The timeout arm then writes the incomplete marker, another `UPDATE + /// pinned_cids`, which blocks on the same lock and elapses against its own floor at + /// ~4.2s. So the repair is reached with its deadline long SPENT, not with ~600ms + /// left, and ~4.2s is the fixture's expected total: one budget plus one chained + /// record grace, which is the `db_record_deadline` re-flooring described on + /// `pin_new_objects`. + /// + /// The test is load-bearing either way, and the 10s wrap is what makes it so: the + /// `pinned_cids` lock is held until after the call returns, so an unbounded + /// `cid_for_oid` inside the repair hangs past the wrap instead of returning here. + #[sqlx::test] + async fn pin_new_objects_stalled_legacy_repair_lookup_returns_by_budget(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("repair_stalled.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + db.record_pinned_cid_with_source(&sha, &seed_cid(), "repo-seed") + .await + .unwrap(); + let endpoint = delaying_endpoint(vec![Duration::ZERO]).await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let mut sources_lock = lock_table(&pool, "pin_repo_sources").await; + + let lock_pool = pool.clone(); + let controller = async { + tokio::time::sleep(Duration::from_millis(1500)).await; + let cids_lock = lock_table(&lock_pool, "pinned_cids").await; + tokio::time::sleep(Duration::from_millis(100)).await; + rollback(&mut sources_lock).await; + cids_lock + }; + + let started = std::time::Instant::now(); + let pin = tokio::time::timeout( + Duration::from_secs(10), + pin_new_objects( + &endpoint, + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-repair-stalled", + Duration::from_millis(2200), + ), + ); + let (pinned, mut cids_lock) = tokio::join!(pin, controller); + pinned.expect( + "the repair's own DB lookup must be bounded by the batch deadline: the bare \ + await hangs past this wrap", + ); + let elapsed = started.elapsed(); + + assert!( + elapsed < Duration::from_secs(6), + "a stall inside repair_legacy_provider_cid must end the call at ~budget \ + (2.2s) plus the one chained record grace the marker write costs against the \ + same lock (~4.2s), never at the lock's lifetime; got {elapsed:?}" + ); + + rollback(&mut cids_lock).await; + } } diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index da76e503..242821f2 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -88,7 +88,7 @@ pub async fn pin_object( /// /// The loop runs under a `pin_semaphore` permit and that pool defers rather than /// sheds, so the hold has to be bounded by something other than the pusher's object -/// count. Two things here are: +/// count. Three things here are: /// /// - this loop's own wall-clock: the deadline is taken once at loop start and /// checked at the top of every iteration, so no object's work begins with less @@ -100,22 +100,51 @@ pub async fn pin_object( /// object's own `git_timeout`, with SIGTERM-then-SIGKILL process-group teardown, so a /// hung `git cat-file` costs this batch one `git_timeout` plus one watchdog teardown /// instead of holding the permit for the child's whole lifetime and blocking a runtime -/// worker while it does. +/// worker while it does; +/// - the DB round-trips: every DB operation reachable from inside the region is +/// bounded by the same absolute deadline through `crate::ipfs_pin::db_bounded`, +/// including the two inside `repair_legacy_provider_cid`, which this loop body's own +/// call sites do not show. `retry_db_record` is wrapped as a whole so its ladder +/// cannot multiply one remainder, and the durability writes (the post-upload +/// `record_pinata_cid` and source record, the skip branch's source record, and both +/// incomplete markers) take the floored remainder `max(remaining, DB_RECORD_GRACE)` +/// so a spent budget delays the permit release rather than dropping a write. That +/// floor matters more here than on the twin: `pinned.push` is unconditional, so a +/// dropped record would leave `api::repos` advertising a CID the resolver 404s. A +/// bound is not a rollback, and what an elapsed bound MEANS is a property of the +/// operation, so each site maps that arm from the shape of the call it wrapped. Both +/// source-record sites wrap `record_pin_source`, an explicit transaction whose +/// `tx.commit()` a cancelled future never reaches, so a timeout there definitely did +/// not land and both write the incomplete marker exactly as the definite-error arm +/// does. `record_pinata_cid` is a single autocommit upsert, so ITS timeout is a +/// genuine unknown outcome and is never treated as a failed write. See +/// `crate::ipfs_pin::BoundedDbError::Elapsed`. /// /// So the LOOP's hold is bounded by roughly `batch_budget`, plus one watchdog -/// teardown and one upload (the shared client's whole-request timeout bounds the -/// upload; `pin_object` takes no per-request override). The PERMIT's hold is NOT -/// bounded by any of this: `api::repos` acquires the permit and then re-derives the -/// object list with `pinata_object_list_for_refs` BEFORE this function is entered, -/// and that walk carries no aggregate deadline. The DB round-trips -/// (`has_pinata_cid`, `record_pinata_cid`) are untimed inside the budgeted region -/// too. +/// teardown, one upload (the shared client's whole-request timeout bounds the +/// upload; `pin_object` takes no per-request override), and the record graces one +/// iteration can chain. `db_record_deadline` re-floors from `Instant::now()` at EVERY +/// call, so the graces inside a single iteration add up rather than sharing one floor: +/// the worst case here is the add path at `deadline + 6s` (`record_pinata_cid`, then +/// `record_pin_source`, then its incomplete marker), against the skip branch's +/// `deadline + 4s`. It does NOT stack per object, because the next iteration's first +/// statement is `batch_budget_gate`, which breaks the batch, so the overrun is one +/// iteration's worth however many objects the push carried. Against the 120s +/// `PIN_BATCH_BUDGET` that is roughly a 5% overrun for the batch, not an unbounded +/// hold. The PERMIT's hold is NOT bounded by any of this, and that stays out of +/// scope: `api::repos` acquires the permit (repos.rs ~2688) and only then re-derives +/// the object list with `pinata_object_list_for_refs` (~2697), BEFORE this function is +/// entered, and that walk carries no aggregate deadline. What is bounded is this +/// loop's own hold, not the permit's total hold and not the semaphore's worst-case +/// queue. /// /// The twin in `ipfs_pin.rs` is at parity with this loop on everything that bounds or /// repairs an object: the shared budget gate, the read bounded by the earlier of the -/// batch deadline and `git_timeout`, and the skip branch's opportunistic legacy -/// provider-CID repair. Change them in lockstep: the skip-if-pinned check, the -/// provenance and source recording, the fault arms, and the budget handling. +/// batch deadline and `git_timeout`, the skip branch's opportunistic legacy +/// provider-CID repair, and the DB bound above, which is the SAME helper and the same +/// floor on both sides rather than a copy. Change them in lockstep: the +/// skip-if-pinned check, the provenance and source recording, the fault arms, and the +/// budget handling. /// /// The RETURNED PAIRS are the one deliberate divergence, and it is not drift. This side /// pushes a pin whose DB record exhausted its retries, because this return is a real @@ -162,16 +191,32 @@ pub async fn pin_new_objects( break; } - match db.has_pinata_cid(&sha).await { + // Every DB call from here to the end of the iteration is bounded by the + // ABSOLUTE batch deadline (F3, #173), through the same `db_bounded` helper the + // ipfs_pin twin routes through: this loop runs under the same global pin permit + // and a bare await parked it for the whole stall. The elapsed arm is mapped per + // site below, never as a blanket "existing error arm": a timeout cancels the + // client future but not the statement Postgres is running, so it reports an + // UNKNOWN outcome, not a failed write. + match crate::ipfs_pin::db_bounded(deadline, db.has_pinata_cid(&sha)).await { Ok(true) => { // Backfill NULL first-pinner provenance from a known source, in lockstep // with the ipfs_pin skip branch: a pinata-only node otherwise leaves // pre-provenance rows' `pinned_cids.repo_id` NULL forever (grok P2-D). The // resolver still finds the object via the pin_repo_sources union below, so // this is a consistency backfill, not a correctness fix. - match db.provenance_for_oid(&sha).await { + // + // Elapsed here is free to skip: the read costs nothing when it lands late, + // and the backfill's own `AND repo_id IS NULL` guard makes a late-landing + // write idempotent. + match crate::ipfs_pin::db_bounded(deadline, db.provenance_for_oid(&sha)).await { Ok(None) => { - if let Err(e) = db.backfill_pin_provenance(&sha, repo_id).await { + if let Err(e) = crate::ipfs_pin::db_bounded( + deadline, + db.backfill_pin_provenance(&sha, repo_id), + ) + .await + { tracing::warn!(sha = %sha, err = %e, "failed to backfill pin provenance"); } } @@ -186,12 +231,60 @@ pub async fn pin_new_objects( // retried through the SHARED helper (this was a bare call, so a single // transient error dropped the source outright) and, on exhaustion, marked // durably so the resolver keeps the bounded scan fallback for the object. - if let Err(e) = - crate::ipfs_pin::retry_db_record(|| db.record_pin_source(&sha, repo_id)).await + // The retry ladder is bounded AS A WHOLE, not per attempt: three stalls + // plus their backoff otherwise multiply one remainder by three. Floored at + // DB_RECORD_GRACE because this is a durability write. + match crate::ipfs_pin::db_bounded( + crate::ipfs_pin::db_record_deadline(deadline), + crate::ipfs_pin::retry_db_record(|| db.record_pin_source(&sha, repo_id)), + ) + .await { - tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); - if let Err(e) = db.mark_pin_sources_incomplete(&sha).await { - tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + Ok(()) => {} + // Elapsed here is a DEFINITE non-write, in lockstep with the twin, + // and for a reason that comes from the operation rather than the + // timeout: `record_pin_source` is an explicit transaction whose + // `tx.commit()` a cancelled future never reaches, so no COMMIT is + // sent and the row cannot have landed. Mark the set incomplete + // exactly as the definite-error arm does; an incomplete-and-unmarked + // set is read as COMPLETE and 404s a copy this repo would serve. The + // marker's cost is bounded (the fallback scan is capped at + // `ipfs_max_legacy_probes` and charges the per-IP work rate limiter + // per probe). The arm stays separate only so the warn tells an + // operator a stalled batch from a scattered per-object failure. + Err(e @ crate::ipfs_pin::BoundedDbError::Elapsed) => { + tracing::warn!( + sha = %sha, + err = %e, + "pin source record did not complete inside the batch deadline; \ + a cancelled multi-statement transaction never commits, so the \ + source is definitely missing and the set is marked incomplete" + ); + if let Err(e) = crate::ipfs_pin::db_bounded( + crate::ipfs_pin::db_record_deadline(deadline), + db.mark_pin_sources_incomplete(&sha), + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } + } + // The retries are spent on REAL errors, so this repo is definitely NOT + // in the source set and the set is known incomplete. Floored for the + // same reason the record above is: a spent budget must not drop the + // compensation. The marker write is a single autocommit statement, so + // ITS own elapsed arm is a genuine unknown outcome; nothing branches + // on it, which is why warn-only is right there. + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + if let Err(e) = crate::ipfs_pin::db_bounded( + crate::ipfs_pin::db_record_deadline(deadline), + db.mark_pin_sources_incomplete(&sha), + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } } } // R8 (#173 round 10), in lockstep with the ipfs_pin skip branch: @@ -311,9 +404,28 @@ pub async fn pin_new_objects( // U3 (#173): both records go through the shared retry helper, at parity // with the ipfs_pin twin. These were bare calls, so one transient DB error // permanently dropped a pin source. - if let Err(e) = crate::ipfs_pin::retry_db_record(|| { - db.record_pinata_cid(&sha, &raw_cid, &cid, Some(repo_id)) - }) + // + // Both bounds here are FLOORED at DB_RECORD_GRACE (F3, #173). The upload + // runs under the shared client's own ceiling, so a successful one can + // return with ~0 of the batch budget left, and an unfloored bound would + // fail a write that today completes in milliseconds. That costs more on + // this side than on the twin: `pinned.push` below is UNCONDITIONAL, so + // `api/repos.rs` builds its `cid_map` from the pair either way and drives + // `upsert_branch_cid` plus the p2p `publish_ref_update` gossip from it. A + // dropped record therefore makes the node ADVERTISE a CID whose `/ipfs` + // read 404s. If the floored bound still fires, THIS site's outcome really + // is unknown, and unlike the source record below that is a property of + // the operation: `record_pinata_cid` is a single autocommit upsert, so + // the statement Postgres already started can still land after the client + // future is cancelled. The warn names the arm through the error's own + // Display and the site keeps its existing behavior: the pair is still + // returned, and the row may or may not exist. + if let Err(e) = crate::ipfs_pin::db_bounded( + crate::ipfs_pin::db_record_deadline(deadline), + crate::ipfs_pin::retry_db_record(|| { + db.record_pinata_cid(&sha, &raw_cid, &cid, Some(repo_id)) + }), + ) .await { tracing::warn!(sha = %sha, err = %e, "failed to record pinata_cid in DB"); @@ -321,12 +433,47 @@ pub async fn pin_new_objects( // F1 (#173 round 8): also record the first pinner in pin_repo_sources. // U3: an exhausted retry marks the set incomplete so the resolver keeps // the scan fallback rather than 404ing a copy it could serve. - if let Err(e) = - crate::ipfs_pin::retry_db_record(|| db.record_pin_source(&sha, repo_id)).await + match crate::ipfs_pin::db_bounded( + crate::ipfs_pin::db_record_deadline(deadline), + crate::ipfs_pin::retry_db_record(|| db.record_pin_source(&sha, repo_id)), + ) + .await { - tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); - if let Err(e) = db.mark_pin_sources_incomplete(&sha).await { - tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + Ok(()) => {} + // Same rule as the skip branch above, and the same reason: this + // wraps `record_pin_source`, an explicit transaction, so a timed-out + // call definitely never committed and the source is definitely + // missing. Mark the set incomplete rather than leaving it incomplete + // and unmarked. Note the contrast with `record_pinata_cid` a few + // lines up: that one is a single autocommit statement, so its + // timeout genuinely is an unknown outcome and it is warn-only. + Err(e @ crate::ipfs_pin::BoundedDbError::Elapsed) => { + tracing::warn!( + sha = %sha, + err = %e, + "pin source record did not complete inside the batch deadline; \ + a cancelled multi-statement transaction never commits, so the \ + source is definitely missing and the set is marked incomplete" + ); + if let Err(e) = crate::ipfs_pin::db_bounded( + crate::ipfs_pin::db_record_deadline(deadline), + db.mark_pin_sources_incomplete(&sha), + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } + } + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + if let Err(e) = crate::ipfs_pin::db_bounded( + crate::ipfs_pin::db_record_deadline(deadline), + db.mark_pin_sources_incomplete(&sha), + ) + .await + { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } } } pinned.push((sha, cid)); @@ -1041,6 +1188,406 @@ mod tests { assert_eq!(requests.load(std::sync::atomic::Ordering::SeqCst), 0); } + // ── Stalled-DB bound (F3, #173) ─────────────────────────────────────── + // + // `api/repos.rs` acquires the GLOBAL `pin_semaphore` for the Pinata + // replication task and holds it across this whole function, the same permit + // the IPFS lane takes. `batch_budget_gate` only gates BETWEEN objects and the + // git read is already clamped, so a bare DB await in the region parked that + // permit for as long as the query was stuck. The tests below drive the stall + // with a `LOCK TABLE .. IN ACCESS EXCLUSIVE MODE` held on a dedicated pooled + // connection, the same technique the ipfs_pin twin's stall tests use, and copy + // their tolerances (a 1.5s budget, an `elapsed < 3s` assertion, a 10s outer + // wrap). The budget is above `PIN_READ_FLOOR` on purpose: below it + // `batch_budget_gate` breaks the batch as the loop body's FIRST statement, so a + // ~1s budget would never reach a DB call and the test would pass with the bound + // deleted. + // --------------------------------------------------------------------- + + /// Take an `ACCESS EXCLUSIVE` lock on `table` on a dedicated pooled connection. + /// Every SELECT needs `ACCESS SHARE`, which conflicts, so the next statement + /// touching the table blocks at lock acquisition regardless of row count. + /// Copied from `ipfs_pin.rs`'s test mod rather than shared, since test mods are + /// private, the same way `seed_loose_blobs` and `capture_logs` are. + async fn lock_table( + pool: &sqlx::PgPool, + table: &str, + ) -> sqlx::pool::PoolConnection { + let mut conn = pool.acquire().await.unwrap(); + sqlx::raw_sql(&format!( + "BEGIN; LOCK TABLE {table} IN ACCESS EXCLUSIVE MODE;" + )) + .execute(&mut *conn) + .await + .unwrap(); + conn + } + + async fn rollback(conn: &mut sqlx::pool::PoolConnection) { + sqlx::raw_sql("ROLLBACK") + .execute(&mut **conn) + .await + .unwrap(); + } + + /// Scenario 3: the FIRST DB call in this lane's budgeted region + /// (`has_pinata_cid`) stalls. With the batch deadline bounding it the loop + /// abandons the object, the budget gate then breaks the batch, and the call + /// returns at ~budget having uploaded nothing. Pre-fix the bare await blocks for + /// the lock's whole lifetime, holding the caller's global pin permit with it. + /// + /// The upload mock is at `.expect(0)`: a stalled pinned-status check must never + /// fall through to an upload, since that would re-send bytes Pinata may already + /// hold and, worse, return a CID this node then advertises. + #[sqlx::test] + async fn pinata_pin_new_objects_stalled_db_returns_by_budget(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("pinata_stalled.git"); + let oids = seed_loose_blobs(&repo_path, 1); + + let mut server = mockito::Server::new_async().await; + let upload = server + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .with_body(r#"{"data":{"cid":"QmShouldNotHappen"}}"#) + .expect(0) + .create_async() + .await; + + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let mut lock = lock_table(&pool, "pinned_cids").await; + + let client = reqwest::Client::new(); + let started = std::time::Instant::now(); + let pinned = tokio::time::timeout( + Duration::from_secs(10), + pin_new_objects( + &client, + &server.url(), + "test-jwt", + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-pinata-stalled", + Duration::from_millis(1500), + ), + ) + .await + .expect( + "a stalled DB must cost the batch its budget, not the lock's lifetime: the \ + bare await hangs past this wrap", + ); + let elapsed = started.elapsed(); + + assert!( + pinned.is_empty(), + "a stalled pinata-status check cannot produce a pinned object: {pinned:?}" + ); + assert!( + elapsed < Duration::from_secs(3), + "the batch deadline must end the call at ~budget (1.5s); got {elapsed:?}" + ); + upload.assert_async().await; + + rollback(&mut lock).await; + } + + /// The Pinata half of the timed-out source record, driven rather than argued: the + /// twin's `pin_new_objects_skip_branch_stalled_record_returns_by_budget` covers the + /// Kubo lane and this covers the site that has to change in lockstep with it. + /// + /// The object already has a `pinata_cid`, so the loop takes the skip branch and + /// tries to record this repo as an additional source; `pin_repo_sources` is locked + /// for the whole run, so that insert stalls inside `retry_db_record` and the whole + /// ladder elapses against one floored remainder. Two properties: + /// + /// - the call still returns promptly, at the record floor rather than the lock's + /// lifetime; + /// - on the TIMEOUT arm the incomplete marker IS written. `record_pin_source` is an + /// explicit transaction, so the cancelled future never reaches `tx.commit()`, no + /// COMMIT is ever sent, and the source definitely did not land. Withholding the + /// marker there would leave the set incomplete AND unmarked, which the resolver + /// reads as complete and 404s a copy this repo would serve. + /// + /// `pinned_cids` is deliberately NOT locked, so the marker write itself is free to + /// land and the assertion below is about the branch, not about lock contention. + #[sqlx::test] + async fn pinata_skip_branch_stalled_record_marks_incomplete(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("pinata_skip_stalled.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + // Seed the object as already Pinata-pinned, by a DIFFERENT repo, so the skip + // branch is taken and the source record below is a genuine additional-source + // insert rather than a no-op on the conflict. The resolver key is a canonical + // raw CIDv1 so the opportunistic legacy repair takes its cost gate and reads no + // bytes. + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"pinata skip seed").to_string(); + db.record_pinata_cid(&sha, &raw_cid, "QmSeedProviderCid", Some("repo-seed")) + .await + .unwrap(); + db.record_pin_source(&sha, "repo-seed").await.unwrap(); + let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let endpoint = delaying_pinata_endpoint( + vec![Duration::from_millis(0)], + std::sync::Arc::clone(&requests), + ) + .await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let mut lock = lock_table(&pool, "pin_repo_sources").await; + + let client = reqwest::Client::new(); + let started = std::time::Instant::now(); + let pinned = tokio::time::timeout( + Duration::from_secs(15), + pin_new_objects( + &client, + &endpoint, + "test-jwt", + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-pinata-skip-stalled", + Duration::from_millis(1500), + ), + ) + .await + .expect( + "the wrapped retry ladder must fit inside one floored remainder: the bare \ + retry_db_record hangs past this wrap", + ); + let elapsed = started.elapsed(); + + assert!( + pinned.is_empty(), + "an already-pinned object is skipped, never re-uploaded: {pinned:?}" + ); + assert_eq!( + requests.load(std::sync::atomic::Ordering::SeqCst), + 0, + "the skip branch must not reach the upload at all" + ); + assert!( + elapsed < Duration::from_secs(5), + "the record's floored remainder must end the call promptly; got {elapsed:?}" + ); + + rollback(&mut lock).await; + drop(lock); + + assert!( + db.pin_sources_incomplete(&sha).await.unwrap(), + "a TIMED-OUT `record_pin_source` definitively did not land: it is an explicit \ + multi-statement transaction, and the cancelled future never reaches \ + `tx.commit()`, so no COMMIT is ever sent and the row cannot exist. The set is \ + therefore incomplete, and leaving it UNMARKED is the exact state the marker \ + exists to prevent: the resolver reads a non-empty below-cap set as complete \ + and 404s a copy this repo would serve" + ); + } + + /// Scenario 8, the Pinata half of the durability floor. `batch_budget_gate` only + /// guarantees `PIN_READ_FLOOR` before an object STARTS and the upload runs under + /// the shared client's own ceiling, so a successful upload can return with ~0 of + /// the batch budget left. Without the floor the post-upload `record_pinata_cid` + /// would then be failed by a spent deadline, and this lane's `pinned.push` is + /// UNCONDITIONAL: `api/repos.rs` builds `cid_map` from the return and drives + /// `upsert_branch_cid` plus the p2p `publish_ref_update` gossip from it, so a + /// dropped record makes the node advertise a CID whose `/ipfs` read 404s. + /// + /// Fixture: a 2s budget, a 1.7s upload, and `pinned_cids` locked from 500ms (well + /// after `has_pinata_cid` has read it, and still well before the upload returns) + /// until 2.4s. The record therefore starts at ~1.72s with ~280ms of budget left + /// and needs ~680ms of lock wait to land, which only the `DB_RECORD_GRACE` floor + /// buys it. + /// + /// The lock time is a MARGIN, not a boundary: taking it at 100ms left + /// `has_pinata_cid` racing it on a loaded box, and losing that race makes the read + /// block, time out, and break the batch, which fails on `pinned.len() == 1` for a + /// reason that has nothing to do with the floor. Any time between the + /// `has_pinata_cid` round trip and the upload's 1.7s return proves the same thing. + #[sqlx::test] + async fn pinata_pin_add_with_spent_budget_still_records_cid(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("pinata_spent_budget.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let endpoint = delaying_pinata_endpoint( + vec![Duration::from_millis(1700)], + std::sync::Arc::clone(&requests), + ) + .await; + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let lock_pool = pool.clone(); + let locker = async move { + tokio::time::sleep(Duration::from_millis(500)).await; + let mut conn = lock_table(&lock_pool, "pinned_cids").await; + tokio::time::sleep(Duration::from_millis(1900)).await; + rollback(&mut conn).await; + }; + + let client = reqwest::Client::new(); + let pin = tokio::time::timeout( + Duration::from_secs(20), + pin_new_objects( + &client, + &endpoint, + "test-jwt", + &repo_path, + "git", + Duration::from_secs(30), + oids.clone(), + &db, + "repo-pinata-spent-budget", + Duration::from_millis(2000), + ), + ); + let (pinned, ()) = tokio::join!(pin, locker); + let pinned = pinned.expect("the floored record must land well inside this wrap"); + + assert_eq!( + requests.load(std::sync::atomic::Ordering::SeqCst), + 1, + "the fixture only proves anything if the upload actually ran" + ); + assert!( + db.has_pinata_cid(&sha).await.unwrap(), + "a successful upload whose batch deadline is spent must still land its \ + pinned_cids row: this lane pushes the pair unconditionally, so a dropped \ + record makes api/repos.rs advertise a CID the resolver cannot serve" + ); + assert_eq!( + pinned.len(), + 1, + "the uploaded pin must still be returned: {pinned:?}" + ); + } + + /// The POST-UPLOAD source record's timeout arm, the one site of the three that + /// nothing else executes. The skip-branch twin above and the ipfs_pin lane cover + /// the other two; this arm sits after a SUCCESSFUL `pin_object`, so no skip-branch + /// fixture can reach it. + /// + /// Why it has to write the marker at all: `record_pin_source` is an explicit + /// transaction ending in `tx.commit()`, and a cancelled future never gets there, so + /// no COMMIT is sent and the row definitely does not exist. The set is therefore + /// incomplete, and leaving it unmarked is the state the marker exists to prevent. + /// + /// Fixture: the object is NOT seeded as Pinata-pinned, so `has_pinata_cid` is false + /// and the run takes the upload path. The mock answers the upload at once, then + /// `pin_repo_sources` is held under `ACCESS EXCLUSIVE` for the whole run, so the + /// post-upload `record_pin_source` blocks and elapses against its floored bound at + /// ~2s. `pinned_cids` is deliberately left UNLOCKED, so both `record_pinata_cid` + /// and the marker write itself are free to land and the assertion is about the arm + /// rather than about my own lock. + /// + /// The upload assertion is what keeps this from being vacuous: without it the test + /// would pass just as well if the run never reached the post-upload path at all. + #[sqlx::test] + async fn pinata_post_upload_stalled_record_marks_incomplete(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("pinata_post_upload.git"); + let oids = seed_loose_blobs(&repo_path, 1); + let sha = oids[0].clone(); + + let mut server = mockito::Server::new_async().await; + let upload = server + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .with_body(r#"{"data":{"cid":"QmPostUploadProviderCid"}}"#) + .expect(1) + .create_async() + .await; + + // Install a log capture even though nothing here asserts on it: `tracing` + // caches a callsite's interest globally the first time it is hit, and a hit + // from a thread with no subscriber caches it as never-interested for the whole + // binary, which silently blinds the sibling tests that DO assert on the batch + // deadline warn. + let (_logs, _log_guard) = capture_logs(); + + let mut lock = lock_table(&pool, "pin_repo_sources").await; + + let client = reqwest::Client::new(); + let started = std::time::Instant::now(); + let pinned = tokio::time::timeout( + Duration::from_secs(20), + pin_new_objects( + &client, + &server.url(), + "test-jwt", + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-pinata-post-upload", + Duration::from_millis(1500), + ), + ) + .await + .expect( + "the wrapped retry ladder must fit inside one floored remainder: the bare \ + retry_db_record hangs past this wrap", + ); + let elapsed = started.elapsed(); + + rollback(&mut lock).await; + drop(lock); + + upload.assert_async().await; + assert_eq!( + pinned.len(), + 1, + "the upload succeeded, so this lane still returns the pair: {pinned:?}" + ); + assert!( + elapsed < Duration::from_secs(8), + "the record's floored remainder must end the call promptly, never at the \ + lock's lifetime; got {elapsed:?}" + ); + assert!( + db.pin_sources_incomplete(&sha).await.unwrap(), + "the POST-UPLOAD arm must mark the source set incomplete when its record \ + times out: `record_pin_source` is an explicit transaction whose cancelled \ + future never reaches `tx.commit()`, so the row definitely did not land, and \ + an incomplete-and-unmarked set is read as complete and 404s a copy this \ + repo would serve" + ); + } + #[tokio::test] async fn test_pin_skipped_when_jwt_empty() { let client = reqwest::Client::new(); From 1b44b12b468c265f946390a57cd815d67d205166 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 11 Aug 2026 04:33:47 -0500 Subject: [PATCH 36/77] fix(node): rewind the pin repair sweep cursor on every completed walk A clean sweep run parked its cursor at the table's maximum sha256_hex and later runs read only rows above it, so a provider-CID row written below that point by another node during a rolling upgrade was never swept. The resolver withholds such a row's advertised key, leaving the object unretrievable with nothing left to repair it. The round 11 rewind could not cover this: it fired only when a run skipped a retryable row, and the run that parks the cursor is by definition clean. The rewind is now unconditional on reaching the end of the table. A run that stops on a pass error keeps its position instead, so a node whose DB fails part-way through does not restart the walk from the beginning on every boot. The cost is one ordered scan per run, plus one repair attempt per run for a row that is unrepairable in principle. Rows already carrying the canonical raw key cost a codec decode and no object read, so a node that has finished repairing pays the scan and nothing more. The terminal-skip test now asserts that bounded cost rather than the row going unread. --- crates/gitlawb-node/src/ipfs_pin.rs | 36 +++-- crates/gitlawb-node/src/test_support.rs | 198 +++++++++++++++++++++++- 2 files changed, 217 insertions(+), 17 deletions(-) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index cd704a33..393642e6 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -434,15 +434,29 @@ pub(crate) async fn sweep_legacy_provider_cids_once( /// or repairing an individual row are warn-and-skip; only a failure of the batch query /// or the cursor write ends the run, and a later run picks up from the stored cursor. /// -/// A run that skipped at least one RETRYABLE row rewinds the cursor to the start of the -/// table on its way out (#173 round 11). Without that the cursor parked at the maximum -/// `sha256_hex` for good: every later boot read zero rows, so a row skipped for a -/// transient reason (its repo cold on a Tigris-backed node, a DB or object read error) -/// was skipped permanently, unadvertised and unresolvable with nothing left to fix it. -/// The rewind is a per-RUN decision made after the walk has already finished, never -/// mid-walk, so it cannot spin: the cost is one extra ordered scan on the next run, and -/// a row that is unrepairable in principle (bytes gone, provenance gone) does not count -/// as retryable, so a node holding one does not re-walk on every boot forever. +/// A run that REACHES THE END OF THE TABLE rewinds the cursor to the start on its way +/// out, so the next run walks the whole table again (#173 rounds 11 and 12). Without +/// that the cursor parked at the maximum `sha256_hex` for good and every later boot +/// read zero rows, which stranded two different kinds of row: one skipped for a +/// transient reason (its repo cold on a Tigris-backed node, a DB or object read error), +/// and one written BELOW the parked cursor afterwards by another node mid-rolling- +/// upgrade. Either way the row was unadvertised and unresolvable with nothing left to +/// fix it. +/// +/// Round 11 gated the rewind on a transient skip having happened. That could not cover +/// the second case, because the run that parks the cursor is a clean one by definition: +/// the row it strands does not exist yet. So the rewind is unconditional on completion. +/// +/// It is a per-RUN decision made after the walk has finished, never mid-walk, so it +/// cannot spin. The cost is one extra ordered scan per run, plus one repair attempt per +/// run for each row that is unrepairable in principle (bytes gone, provenance gone) — +/// the read is attempted before the bytes are found missing. A row already carrying the +/// canonical raw key costs a codec decode and no read at all, so a node that has +/// finished repairing pays the scan and nothing more. +/// +/// A run that stops on a pass ERROR does NOT rewind: its cursor is mid-table and +/// discarding it would restart the walk from the beginning on a node whose DB is +/// failing part-way through. pub(crate) async fn sweep_legacy_provider_cids( repos_dir: &std::path::Path, git_bin: &str, @@ -452,6 +466,7 @@ pub(crate) async fn sweep_legacy_provider_cids( db: &crate::db::Db, ) -> SweepStats { let mut totals = SweepStats::default(); + let mut completed = false; loop { let pass = match sweep_pass(repos_dir, git_bin, git_timeout, batch, db).await { Ok(p) => p, @@ -467,11 +482,12 @@ pub(crate) async fn sweep_legacy_provider_cids( // A short batch means the ordered walk reached the end of the table. Stop here // rather than after an extra empty pass, and do NOT sleep on the way out. if (pass.scanned as i64) < batch { + completed = true; break; } tokio::time::sleep(delay).await; } - if totals.retryable_skips > 0 { + if completed { if let Err(e) = db.set_pin_repair_cursor("").await { tracing::warn!(err = %e, "failed to rewind the legacy provider-CID sweep cursor"); } diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 30d6325f..733cfa3a 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -6653,10 +6653,19 @@ mod tests { } /// U4 scenario 10 (#173, the other arm of scenario 9): a PERMANENTLY unrepairable - /// row must not make the sweep re-walk forever. Bytes that are genuinely gone are a - /// terminal skip, so the cursor stays parked and a later run reads nothing. Without - /// that split the transient-skip reset of scenario 9 turns every boot on such a node - /// into a full table walk. Both runs are timeout-bounded, so a hot loop FAILS here. + /// row must not cost anything on a later run. Bytes that are genuinely gone stay + /// gone, so a re-walk must not read object bytes for that row, must not repair it, + /// and must not spin: both runs are timeout-bounded, so a hot loop FAILS here. + /// + /// The assertion is about BOUNDED cost, not about the row going unread (jatmn + /// round 12). It asserted `scanned == 0` while the cursor parked at the table + /// maximum on a clean run; that parking is what let a row written below the cursor + /// by another node go unswept forever, so the run now always rewinds on clean + /// completion. The terminal row is therefore re-walked once per run, and its + /// repair is re-attempted once: the object read is attempted before the bytes are + /// found missing. That cost is real and it is the price of D. What must stay true + /// is that it is exactly ONE attempt per run and never repairs, so a regression + /// that retries the dead row within a run fails here. #[sqlx::test] async fn sweep_does_not_rewalk_for_a_terminal_skip(pool: PgPool) { use gitlawb_core::identity::Keypair; @@ -6704,6 +6713,7 @@ mod tests { "the row is walked and cannot be repaired" ); + crate::ipfs_pin::reset_legacy_repair_reads(); let second = tokio::time::timeout( std::time::Duration::from_secs(30), crate::ipfs_pin::sweep_legacy_provider_cids( @@ -6718,9 +6728,183 @@ mod tests { .await .expect("the second run terminates"); assert_eq!( - (second.scanned, second.repaired, second.passes), - (0, 0, 1), - "a terminal skip leaves the cursor parked: the next run re-reads nothing" + (second.repaired, second.passes), + (0, 1), + "the terminal row is still unrepairable and the run does not spin" + ); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 1, + "the dead row costs exactly one repair attempt per run, never a retry loop" + ); + } + + /// U4 (#173, jatmn round 12): a row inserted BELOW a parked cursor must still be + /// swept. A clean run (no retryable skips) leaves the cursor at the table's maximum + /// `sha256_hex` and every later pass reads only `> cursor`, so a provider-CID row + /// written afterwards by an older node mid-rolling-upgrade whose oid sorts below + /// that maximum is never revisited. The resolver withholds its advertised key, so + /// the object stays unretrievable with nothing left to fix it. The rewind added for + /// the transient-skip case does not cover this: it is gated on `retryable_skips > 0` + /// and a clean pass reports zero. + #[sqlx::test] + async fn sweep_revisits_a_row_written_below_a_parked_cursor(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["rollsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("rollsrc.git"); + let repo = seed_repo(&owner_did, "rollsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // Two objects from the fixture, ordered by the column the walk is keyed on. + let mut oids = [fx.public_oid.clone(), fx.secret_oid.clone()]; + oids.sort(); + let (low_oid, high_oid) = (oids[0].clone(), oids[1].clone()); + + // First boot: one legacy row, repaired, nothing retryable. The cursor parks at + // that row's oid, which is the table maximum. + seed_legacy_pin(&pool, &bare, &high_oid, Some(&repo.id)).await; + let first = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the first run terminates"); + assert_eq!( + (first.repaired, first.retryable_skips), + (1, 0), + "the first run is a clean completion, so no rewind is triggered" + ); + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + "", + "a completed run rewinds instead of parking at the table maximum" + ); + + // An older node in the rolling upgrade writes a provider-CID row that sorts + // below the parked cursor. + let (low_raw, low_provider) = seed_legacy_pin(&pool, &bare, &low_oid, Some(&repo.id)).await; + + // Next boot. + let second = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the second run terminates"); + + let (stored, stashed) = stored_pin(&pool, &low_oid).await; + assert_eq!( + stored, low_raw, + "the row written below the cursor is repaired to the raw-content key \ + (stored {stored}, provider key {low_provider}, second run scanned \ + {} repaired {})", + second.scanned, second.repaired + ); + assert_eq!( + stashed.as_deref(), + Some(low_provider.as_str()), + "its old provider CID is stashed" + ); + assert!( + state + .db + .list_pinned_cids() + .await + .unwrap() + .iter() + .any(|r| r.cid == low_raw), + "the repaired row is advertised again" + ); + } + + /// U4 (#173, round 12, the other side of the unconditional rewind): a run that stops + /// on a pass ERROR keeps its mid-table cursor. The rewind is what a COMPLETED walk + /// does; applying it to a failed one would restart from the beginning of the table + /// on every boot of a node whose DB fails part-way through, and such a node would + /// never reach the rows behind the failure point. The error is induced by renaming + /// `pinned_cids` out from under the walk during the inter-batch sleep. + #[sqlx::test] + async fn sweep_keeps_its_cursor_when_a_pass_fails(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["failsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("failsrc.git"); + let repo = seed_repo(&owner_did, "failsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let mut oids = [fx.public_oid.clone(), fx.secret_oid.clone()]; + oids.sort(); + for oid in &oids { + seed_legacy_pin(&pool, &bare, oid, Some(&repo.id)).await; + } + + // A batch of one means the first pass is full, so the run sleeps and comes back + // for a second pass. The table is gone by then. + let killer = { + let pool = pool.clone(); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + sqlx::query("ALTER TABLE pinned_cids RENAME TO pinned_cids_gone") + .execute(&pool) + .await + .expect("rename the table out from under the walk"); + }) + }; + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 1, + std::time::Duration::from_millis(300), + &state.db, + ), + ) + .await + .expect("the run terminates on the failed pass"); + killer.await.expect("the killer task completes"); + + assert_eq!( + stats.scanned, 1, + "the first pass read its one row before the table went away" + ); + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + oids[0], + "a failed run keeps the position it reached instead of rewinding" ); } From e4aaae8c5229355010ca710abf22c181210201c3 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:11:34 -0500 Subject: [PATCH 37/77] fix(node): bound the fruitless reads one sweep run will spend Rewinding on every completed walk means a later run re-attempts the object read for every row whose bytes are permanently gone, so a node that collected dead pins from a deleted repo or a force-pushed history paid O(dead rows) git invocations on every boot with no decay. A run now stops after 64 such reads and keeps its cursor, so the next boot resumes past what it already walked and the table is still covered across boots. Also corrects two pieces of prose that described the superseded round 11 behavior: the SweepStats::retryable_skips field doc still claimed that field drives the rewind, and the new below-cursor test's comments still said a clean run parks its cursor. The pass-failure test waited on a fixed 100ms sleep for the first pass to finish before renaming the table out from under the walk. It now waits on the cursor that pass writes, so a runner slow enough to push the first pass past the sleep no longer fails the assertion for the wrong reason. --- crates/gitlawb-node/src/ipfs_pin.rs | 63 ++++++++++++-- crates/gitlawb-node/src/test_support.rs | 106 ++++++++++++++++++++++-- 2 files changed, 155 insertions(+), 14 deletions(-) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 393642e6..77d0155e 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -267,13 +267,33 @@ pub(crate) struct SweepStats { pub repaired: usize, pub passes: usize, /// Rows left unrepaired for a reason a LATER run could fix (the source repo is not - /// on this node's local disk, a DB read failed, a bounded object read failed). A - /// nonzero count is what makes the run rewind its cursor instead of parking it at - /// the end of the table forever. Rows that are unrepairable in principle (no - /// provenance, the repo row is gone, the bytes are gone) are NOT counted here. + /// on this node's local disk, a DB read failed, a bounded object read failed). Rows + /// that are unrepairable in principle (no provenance, the repo row is gone, the + /// bytes are gone) are NOT counted here. + /// + /// This drives NO control decision. It gated the cursor rewind under round 11; the + /// rewind now fires on reaching the end of the table, whatever happened on the way + /// (see [`sweep_legacy_provider_cids`]). Re-gating it on this field reopens the + /// below-cursor rolling-upgrade hole, because the run that parks the cursor is a + /// clean one by definition. The field is reporting only. pub retryable_skips: usize, + /// Object reads spent on rows that turned out to be unrepairable in principle: the + /// bytes are gone, so the read is pure waste and the next run will waste it again. + /// [`MAX_DEAD_ROW_READS_PER_RUN`] bounds this per run. + pub dead_row_reads: usize, } +/// How many fruitless object reads one sweep run will spend before it stops and leaves +/// the rest of the table for the next run (#173 round 12, second-model pass). +/// +/// A completed run rewinds, so every later run re-attempts the read for every row whose +/// bytes are permanently gone. Without a bound that is `O(dead rows)` `git cat-file` +/// invocations on every single boot, and a node that accumulated a lot of them (a +/// deleted repo, a force-pushed history, a failed migration) pays it forever. Stopping +/// early keeps the cursor, so the next boot resumes past the rows already walked rather +/// than repeating them, and the table still gets covered across boots. +pub(crate) const MAX_DEAD_ROW_READS_PER_RUN: usize = 64; + /// One bounded pass of the U4 sweep: read at most `batch` `pinned_cids` rows after /// the persisted cursor, repair the legacy ones, and persist the new cursor. /// @@ -297,6 +317,7 @@ async fn sweep_pass( let scanned = rows.len(); let mut repaired = 0usize; let mut retryable_skips = 0usize; + let mut dead_row_reads = 0usize; let mut last = cursor; for (sha, stored) in rows { @@ -324,6 +345,9 @@ async fn sweep_pass( // along the way was a transient obstacle rather than a permanent one. let mut row_repaired = false; let mut row_retryable = false; + // Whether any source got as far as spending an object read on this row, which is + // what makes an unrepairable row COST something rather than just being skipped. + let mut row_read_attempted = false; for repo_id in sources { let repo = match db.get_repo_by_id(&repo_id).await { Ok(Some(r)) => r, @@ -366,6 +390,7 @@ async fn sweep_pass( } // The sweep holds no pin permit and has no batch to overrun, so the plain // `git_timeout` is the right budget here. + row_read_attempted = true; match repair_legacy_provider_cid( &repo_path, git_bin, @@ -394,6 +419,11 @@ async fn sweep_pass( if !row_repaired && row_retryable { retryable_skips += 1; } + // Read, not repaired, and nothing a later run would change: pure waste, and the + // rewind means the next run repeats it. This is the quantity the run bounds. + if row_read_attempted && !row_repaired && !row_retryable { + dead_row_reads += 1; + } } db.set_pin_repair_cursor(&last).await?; @@ -402,6 +432,7 @@ async fn sweep_pass( repaired, passes: 1, retryable_skips, + dead_row_reads, }) } @@ -448,11 +479,13 @@ pub(crate) async fn sweep_legacy_provider_cids_once( /// the row it strands does not exist yet. So the rewind is unconditional on completion. /// /// It is a per-RUN decision made after the walk has finished, never mid-walk, so it -/// cannot spin. The cost is one extra ordered scan per run, plus one repair attempt per -/// run for each row that is unrepairable in principle (bytes gone, provenance gone) — -/// the read is attempted before the bytes are found missing. A row already carrying the -/// canonical raw key costs a codec decode and no read at all, so a node that has -/// finished repairing pays the scan and nothing more. +/// cannot spin. The cost is one extra ordered scan per run, plus a repair attempt for +/// each row that is unrepairable in principle (bytes gone, provenance gone): the read is +/// attempted before the bytes are found missing. Those reads are the one cost that does +/// not shrink as the migration progresses, so `MAX_DEAD_ROW_READS_PER_RUN` bounds them +/// per run and the run stops early rather than paying `O(dead rows)` on every boot. A +/// row already carrying the canonical raw key costs a codec decode and no read at all, +/// so a node that has finished repairing pays the scan and nothing more. /// /// A run that stops on a pass ERROR does NOT rewind: its cursor is mid-table and /// discarding it would restart the walk from the beginning on a node whose DB is @@ -478,6 +511,7 @@ pub(crate) async fn sweep_legacy_provider_cids( totals.scanned += pass.scanned; totals.repaired += pass.repaired; totals.retryable_skips += pass.retryable_skips; + totals.dead_row_reads += pass.dead_row_reads; totals.passes += 1; // A short batch means the ordered walk reached the end of the table. Stop here // rather than after an extra empty pass, and do NOT sleep on the way out. @@ -485,6 +519,17 @@ pub(crate) async fn sweep_legacy_provider_cids( completed = true; break; } + // Enough fruitless reads for one run. Stop WITHOUT completing, so the cursor + // stays where the walk got to and the next run carries on from there instead of + // re-reading these rows. Checked between passes, so a run can overshoot by at + // most one batch. + if totals.dead_row_reads >= MAX_DEAD_ROW_READS_PER_RUN { + tracing::info!( + dead_row_reads = totals.dead_row_reads, + "legacy provider-CID sweep pausing: too many unrepairable rows this run" + ); + break; + } tokio::time::sleep(delay).await; } if completed { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 733cfa3a..23f522c9 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -6769,8 +6769,9 @@ mod tests { oids.sort(); let (low_oid, high_oid) = (oids[0].clone(), oids[1].clone()); - // First boot: one legacy row, repaired, nothing retryable. The cursor parks at - // that row's oid, which is the table maximum. + // First boot: one legacy row, repaired, nothing retryable. Under round 11 this + // is exactly the run that parked the cursor at that row's oid, the table + // maximum, because a clean run reported no retryable skip to rewind for. seed_legacy_pin(&pool, &bare, &high_oid, Some(&repo.id)).await; let first = tokio::time::timeout( std::time::Duration::from_secs(30), @@ -6788,7 +6789,7 @@ mod tests { assert_eq!( (first.repaired, first.retryable_skips), (1, 0), - "the first run is a clean completion, so no rewind is triggered" + "the first run is a clean completion: nothing retryable to rewind for" ); assert_eq!( state.db.pin_repair_cursor().await.unwrap(), @@ -6797,7 +6798,7 @@ mod tests { ); // An older node in the rolling upgrade writes a provider-CID row that sorts - // below the parked cursor. + // below where the walk finished, which is where round 11 left the cursor. let (low_raw, low_provider) = seed_legacy_pin(&pool, &bare, &low_oid, Some(&repo.id)).await; // Next boot. @@ -6840,6 +6841,89 @@ mod tests { ); } + /// U4 (#173, round 12, second-model pass): the fruitless reads a run spends on rows + /// whose bytes are permanently gone are bounded per run. The rewind means every + /// later run re-attempts each of them, so without a bound a node that accumulated + /// dead pins (a deleted repo, a force-pushed history) pays `O(dead rows)` git + /// invocations on every boot, forever. The run stops early instead and keeps its + /// cursor, so the next boot resumes past what it already walked. + #[sqlx::test] + async fn sweep_bounds_fruitless_reads_per_run(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_DEAD_ROW_READS_PER_RUN; + let batch: i64 = 16; + + // A real repo on disk, so every row gets as far as spending an object read, and + // objects that were never in it, so every one of those reads is wasted. + let _fx = seed_cid_repos(&slug, &short, &["deadsrc"]); + let repo = seed_repo(&owner_did, "deadsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + let dead_rows = cap + 2 * batch as usize; + for i in 0..dead_rows { + let phantom_oid = format!("{:064x}", i); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) \ + VALUES ($1, $2, $3, $4)", + ) + .bind(&phantom_oid) + .bind(legacy_dagpb_cid(&raw_cid)) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + } + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + batch, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the run terminates"); + + assert!( + stats.dead_row_reads >= cap, + "the run spends its budget before stopping (spent {})", + stats.dead_row_reads + ); + assert!( + stats.dead_row_reads < cap + batch as usize, + "the run overshoots its budget by at most one batch (spent {}, cap {cap})", + stats.dead_row_reads + ); + assert!( + stats.scanned < dead_rows, + "the run stops short of the table (scanned {} of {dead_rows})", + stats.scanned + ); + + // Not a completed walk, so the cursor is kept and the next run carries on from + // it rather than re-reading the rows this one already paid for. + let cursor = state.db.pin_repair_cursor().await.unwrap(); + assert_ne!(cursor, "", "a run that stops on its budget keeps its place"); + let resumed = state.db.pinned_cids_after(&cursor, batch).await.unwrap(); + assert_eq!( + resumed.first().map(|(sha, _)| sha.as_str()), + Some(format!("{:064x}", stats.scanned).as_str()), + "the next run starts at the row after the last one walked" + ); + } + /// U4 (#173, round 12, the other side of the unconditional rewind): a run that stops /// on a pass ERROR keeps its mid-table cursor. The rewind is what a COMPLETED walk /// does; applying it to a failed one would restart from the beginning of the table @@ -6871,10 +6955,22 @@ mod tests { // A batch of one means the first pass is full, so the run sleeps and comes back // for a second pass. The table is gone by then. + // + // The killer WAITS for the first pass to finish rather than racing a fixed sleep + // against it: the pass writes its cursor as its last act, so a non-empty cursor + // is the signal that the run is now in its inter-batch sleep. A fixed delay here + // fails on a runner slow enough that the rename lands during the first pass's + // own query, which reports `scanned = 0` and asserts something else entirely. let killer = { let pool = pool.clone(); + let db = state.db.clone(); tokio::spawn(async move { - tokio::time::sleep(std::time::Duration::from_millis(100)).await; + loop { + if !db.pin_repair_cursor().await.unwrap().is_empty() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } sqlx::query("ALTER TABLE pinned_cids RENAME TO pinned_cids_gone") .execute(&pool) .await From ed6355ed2c42bed09dec35d46904805e97de806c Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:38:59 -0500 Subject: [PATCH 38/77] fix(node): stop the /ipfs size probe reporting a fault as a missing object object_size_bounded mapped every non-timeout failure to Ok(None), which gate_and_serve reads as a verified absence and does not taint the search for. A corrupt object, an unreadable pack, or a failed spawn therefore ended the search cleanly and handed an authorized caller a definitive 404 instead of the retryable 503 tail, even though the type probe had already reported the object present. Confirmed by execution against the old code: a corrupt loose object returned Ok(None) where the healthy read returned Ok(Some(n)). The probe now returns Result, the vocabulary the type and content stages either side of it already use: a reaped child is Transient, and everything else is classified by store readability, Transient when the store cannot be read and Deterministic when it can. There is no absence value left, so the collapse is unrepresentable rather than fixed at one call site, and the gc-between-stages case is classified like any other fault. ServedRead::Gone lost its only producer and is removed with it. Covered by two store tests, each asserting a healthy read first so the fault case cannot pass on a broken fixture. Not covered: the handler path that turns the fault into a 503, because that call site hardcodes "git" rather than state.git_bin, so no shim can be injected there. --- crates/gitlawb-node/src/api/ipfs.rs | 17 +-- crates/gitlawb-node/src/git/store.rs | 168 ++++++++++++++++++++++++++- 2 files changed, 167 insertions(+), 18 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 05311dc7..d8122663 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -707,10 +707,6 @@ enum ServedRead { Mismatch(String), /// The object exceeds the served-object size cap; withhold rather than buffer it. TooLarge(u64), - /// The object is genuinely absent (git reported it does not exist); try the next - /// candidate. Distinct from `ReadErr` so an infra failure is never silently rendered - /// as a clean not-found. - Gone, /// A git subprocess failed to run (spawn/IO error, not a "no such object"). Logged at /// the handler layer and skipped — an infra failure must surface as an error, not a /// silent 404 for an authorized caller (INV-25 spirit, #173). @@ -1220,13 +1216,11 @@ async fn gate_and_serve( // returns, even if the handler future was dropped or this closure panics. let _admission = read_admission; match store::object_size_bounded(&git_bin, &read_repo, &read_sha, read_deadline) { - Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), - Ok(Some(_)) => {} - // git ran and reported no such object (or an unparseable size): genuine - // not-found for this candidate. - Ok(None) => return ServedRead::Gone, - // git failed to run OR the bounded read timed out (GitServiceTimeout): an - // infra/timeout failure, not a not-found. + Ok(size) if size > max_bytes => return ServedRead::TooLarge(size), + Ok(_) => {} + // Every failure of this stage is a fault, never a not-found: the type probe + // above already returned Present, and the probe no longer has an absence + // value to collapse a corrupt object or a failed spawn into (#173 round 12). Err(e) => return ServedRead::ReadErr(e.to_string()), } let content = match store::read_object_content_bounded( @@ -1272,7 +1266,6 @@ async fn gate_and_serve( ); return GateOutcome::Skip; } - ServedRead::Gone => return GateOutcome::Skip, ServedRead::ReadErr(e) => { // Infra failure (git spawn/IO), NOT a not-found: mark the search truncated so // a wholly-unserved request tails to a retryable 503, never a definitive 404 diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 78a06c89..5617b419 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -606,8 +606,20 @@ pub(crate) fn object_store_readable_store_wide(repo_path: &Path) -> bool { /// is rejected before it is buffered, #173 F6), under /// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git) so a wedged size /// read is reaped at `deadline` instead of pinning the held /ipfs walk admission. -/// `Ok(Some(n))` on success, `Ok(None)` when the object is absent (a non-timeout -/// non-zero exit), `Err(GitServiceTimeout)` on the deadline. +/// `Ok(n)` on success, and otherwise a [`ProbeError`] in the same vocabulary the type and +/// content stages use: a reaped child is `Transient` (retryable), and every other failure +/// goes through [`classify_store_fault`], so it is `Transient` when the object store is +/// not readable and `Deterministic` when it is. +/// +/// There is deliberately NO absence value (#173 round 12). This returned +/// `Ok(None)` for every non-timeout failure, which `gate_and_serve` reads as a verified +/// absence and does not taint the search for, so a corrupt object, an unreadable pack, or +/// a failed spawn handed an authorized caller a definitive 404 instead of the retryable +/// 503 tail. Absence is also not this stage's question: the caller has already had a +/// `Present` verdict from [`object_type_bounded`], so an object that cannot be sized here +/// is a fault, not a not-found, and the one honest exception (a concurrent gc between the +/// two stages) is classified by store readability like any other. Making the absence +/// unrepresentable is what keeps the next caller from reintroducing the swallow. /// /// Takes the shared `deadline` rather than its own timeout so a caller that pairs this /// size check with a later read spends ONE budget across the pair, not one per stage. @@ -616,7 +628,7 @@ pub fn object_size_bounded( repo_path: &Path, sha256_hex: &str, deadline: std::time::Instant, -) -> Result> { +) -> std::result::Result { match crate::git::visibility_pack::run_bounded_git( git_bin, &["cat-file", "-s", sha256_hex], @@ -624,9 +636,25 @@ pub fn object_size_bounded( b"", deadline, ) { - Ok(out) => Ok(String::from_utf8_lossy(&out).trim().parse::().ok()), - Err(e) if e.is::() => Err(e), - Err(_) => Ok(None), + Ok(out) => { + let text = String::from_utf8_lossy(&out); + text.trim().parse::().map_err(|e| { + // git exited 0 but did not print a size: the store answered something + // this code cannot read, which is a fault and never an absence. + classify_store_fault( + repo_path, + sha256_hex, + anyhow::anyhow!("unparseable `cat-file -s` output {:?}: {e}", text.trim()), + ) + }) + } + // The watchdog reaped the child at `deadline`; retryable whatever the store looks + // like, so it is routed before readability gets a say (same rule as the content + // stage in `read_object_bounded`). + Err(e) if e.is::() => { + Err(ProbeError::Transient(e)) + } + Err(e) => Err(classify_store_fault(repo_path, sha256_hex, e)), } } @@ -1488,6 +1516,134 @@ mod tests { ); } + /// #173 round 12 (jatmn): the SIZE probe must use the same absence-versus-fault + /// vocabulary the type probe does. It mapped every non-timeout failure to `Ok(None)`, + /// which `gate_and_serve` reads as a verified absence and does not taint, so a + /// corrupt object, an unreadable pack, or a failed spawn ended the search cleanly and + /// handed an authorized caller a definitive 404 instead of the retryable 503 tail. + /// + /// A corrupt loose object is the same fixture the type stage uses for this, and it is + /// the honest case: the object EXISTS, the type probe says so, and only the size read + /// fails. RED before the change (`Ok(None)`). + #[cfg(unix)] + #[test] + fn object_size_bounded_corrupt_loose_object_is_fault_not_absence() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("sizecorrupt"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."]); + g(&["config", "user.email", "t@t"]); + g(&["config", "user.name", "t"]); + std::fs::write(work.join("f.txt"), b"loose object content\n").unwrap(); + g(&["add", "f.txt"]); + g(&["commit", "-qm", "c1"]); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:f.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + // Healthy first: the probe reads the real size, so the fault case below is not + // passing for the trivial reason that this fixture never worked. + let healthy = super::object_size_bounded("git", &work, &blob, deadline); + assert!( + matches!(healthy, Ok(n) if n > 0), + "a readable object reports its size; got {healthy:?}" + ); + + let obj = work.join(".git/objects").join(&blob[0..2]).join(&blob[2..]); + let mut perms = std::fs::metadata(&obj).unwrap().permissions(); + perms.set_mode(0o644); + std::fs::set_permissions(&obj, perms).unwrap(); + std::fs::write(&obj, b"garbage not a zlib stream").unwrap(); + + let res = super::object_size_bounded("git", &work, &blob, deadline); + assert!( + res.is_err(), + "a corrupt object must surface as a probe fault, never as an absence the \ + resolver renders as a clean 404; got {res:?}" + ); + } + + /// #173 round 12 (jatmn), the transient arm: a store this process cannot read is the + /// retryable case, so the size probe classifies it `Transient` exactly as the type + /// probe does. Distinguishing the two arms is the whole point of routing through + /// `classify_store_fault` rather than returning one undifferentiated error. + #[cfg(unix)] + #[test] + fn object_size_bounded_unreadable_store_is_transient() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("sizeunreadable"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."]); + g(&["config", "user.email", "t@t"]); + g(&["config", "user.name", "t"]); + std::fs::write(work.join("f.txt"), b"loose object content\n").unwrap(); + g(&["add", "f.txt"]); + g(&["commit", "-qm", "c1"]); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:f.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + + // Make this oid's loose fan-out unreadable: git fails, and the store cannot + // certify absence, so the fault is retryable rather than terminal. + let fanout = work.join(".git/objects").join(&blob[0..2]); + let mut perms = std::fs::metadata(&fanout).unwrap().permissions(); + perms.set_mode(0o000); + std::fs::set_permissions(&fanout, perms).unwrap(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let res = super::object_size_bounded("git", &work, &blob, deadline); + + let mut restore = std::fs::metadata(&fanout).unwrap().permissions(); + restore.set_mode(0o755); + std::fs::set_permissions(&fanout, restore).unwrap(); + + assert!( + matches!(res, Err(super::ProbeError::Transient(_))), + "an unreadable object store is the retryable arm; got {res:?}" + ); + } + /// #174 F5/U4: a corrupt LOOSE object makes `git cat-file --batch-check` print /// ` missing` on stdout (exit 0) yet emit `error:` diagnostics on stderr. The /// clean-`missing` absence path must NOT fire here — the `error:` line disqualifies From 5575da4b65ff19da78a6b218da07f854706a811c Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:20:39 -0500 Subject: [PATCH 39/77] fix(node): make the pin-source incompleteness marker per (object, repo) pinned_cids.pin_sources_incomplete was one boolean per object, so any successful source record cleared it, including a genuine record from a repo unrelated to the failure. The resolver reads a cleared marker as "every source is recorded", drops the scan fallback, and 404s an anonymous caller whose only servable copy is the one whose record failed. The missing source is a property of an (object, repo) pair, so v24 stores it as one and only a later record from that same repo clears it. Pre-upgrade markers carry over with an empty sentinel repo id that no real record matches, so they keep their fallback instead of being cleared by the next unrelated record. pinned_cids.pin_sources_incomplete is left in place, unread, so a rollback to the previous release still finds its markers. Also adds the handler-level coverage the size-probe commit could not: a test seam drops the object between the type probe and the size probe, and the request tails to a retryable 503 rather than a definitive 404. The seam is keyed on (repo path, oid) because fixture oids are content-derived and shared across tests, and an oid-only key deleted objects out from under two unrelated tests running in parallel. --- crates/gitlawb-node/src/api/ipfs.rs | 56 +++++++ crates/gitlawb-node/src/db/mod.rs | 116 +++++++++----- crates/gitlawb-node/src/ipfs_pin.rs | 4 +- crates/gitlawb-node/src/pinata.rs | 8 +- crates/gitlawb-node/src/test_support.rs | 193 +++++++++++++++++++++++- 5 files changed, 334 insertions(+), 43 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index d8122663..f9b84171 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -1215,6 +1215,8 @@ async fn gate_and_serve( // Admission clone (#174 U1): the slot stays taken until this blocking work // returns, even if the handler future was dropped or this closure panics. let _admission = read_admission; + #[cfg(test)] + break_size_probe_if_armed(&read_repo, &read_sha); match store::object_size_bounded(&git_bin, &read_repo, &read_sha, read_deadline) { Ok(size) if size > max_bytes => return ServedRead::TooLarge(size), Ok(_) => {} @@ -1314,6 +1316,60 @@ pub async fn list_pins(State(state): State) -> Result>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +fn size_probe_seam_key(repo_path: &std::path::Path, sha256_hex: &str) -> String { + format!("{}::{sha256_hex}", repo_path.display()) +} + +/// Arm the seam: the next serve read of `sha256_hex` FROM `repo_path` loses its loose +/// object between the type probe and the size probe, so the size probe fails on an object +/// git just confirmed present. +#[cfg(test)] +pub(crate) fn break_size_probe_for(repo_path: &std::path::Path, sha256_hex: &str) { + SIZE_PROBE_BREAKERS + .get_or_init(Default::default) + .lock() + .expect("size-probe seam mutex") + .insert(size_probe_seam_key(repo_path, sha256_hex)); +} + +#[cfg(test)] +fn break_size_probe_if_armed(repo_path: &std::path::Path, sha256_hex: &str) { + let armed = SIZE_PROBE_BREAKERS.get().is_some_and(|s| { + s.lock() + .expect("size-probe seam mutex") + .remove(&size_probe_seam_key(repo_path, sha256_hex)) + }); + if armed { + let _ = std::fs::remove_file( + repo_path + .join("objects") + .join(&sha256_hex[0..2]) + .join(&sha256_hex[2..]), + ); + } +} + thread_local! { static PRELOAD_QUERIES: std::cell::Cell = const { std::cell::Cell::new(0) }; } diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 6857667a..0d1c8140 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1011,6 +1011,40 @@ const MIGRATIONS: &[Migration] = &[ )", ], }, + Migration { + version: 24, + name: "pin_source_failures", + stmts: &[ + // #173 round 12 (jatmn): v22's `pin_sources_incomplete` is one boolean per + // OBJECT, so any successful source record cleared it, including one from a + // repo unrelated to the failure. The resolver then read the set as fully + // enumerated, dropped the scan fallback, and 404'd an anonymous caller whose + // only servable copy was the unrecorded public one. The missing source is a + // property of an (object, repo) PAIR, so it is stored as one. + // + // NEW versioned migration (never appended to an applied block, INV-7). A new + // table rather than a column on `pinned_cids`: the relation is many-per-object + // and `CREATE TABLE` takes no lock on the pin table a live node is reading. + "CREATE TABLE IF NOT EXISTS pin_source_failures ( + sha256_hex TEXT NOT NULL, + repo_id TEXT NOT NULL, + PRIMARY KEY (sha256_hex, repo_id) + )", + // Carry the pre-upgrade markers over. Which repo failed was never recorded, + // so they get the empty sentinel, which no real `repo_id` equals: those + // objects keep the scan fallback until something repairs them, rather than + // being cleared by the next unrelated record the way they would have been + // before. Strictly safer than the behavior being replaced, and bounded by how + // rare an exhausted record is. + "INSERT INTO pin_source_failures (sha256_hex, repo_id) + SELECT sha256_hex, '' FROM pinned_cids WHERE pin_sources_incomplete + ON CONFLICT DO NOTHING", + // `pinned_cids.pin_sources_incomplete` is deliberately NOT dropped. Nothing + // reads it after this migration, and leaving it costs one unused boolean, + // whereas dropping it makes a rollback to the previous release lose the + // markers it still reads. + ], + }, ]; /// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). @@ -2827,13 +2861,14 @@ impl Db { .await? .rows_affected(); if inserted > 0 { - sqlx::query( - "UPDATE pinned_cids SET pin_sources_incomplete = FALSE - WHERE sha256_hex = $1 AND pin_sources_incomplete", - ) - .bind(sha256_hex) - .execute(&mut *tx) - .await?; + // Clears THIS repo's failure only (#173 round 12). A boolean per object meant + // repo C's genuine record wiped the marker repo B's failure set, and the + // resolver then dropped the scan fallback while B's copy was still unrecorded. + sqlx::query("DELETE FROM pin_source_failures WHERE sha256_hex = $1 AND repo_id = $2") + .bind(sha256_hex) + .bind(repo_id) + .execute(&mut *tx) + .await?; } tx.commit().await?; Ok(()) @@ -2884,44 +2919,55 @@ impl Db { .await? .rows_affected(); if inserted > 0 { - sqlx::query( - "UPDATE pinned_cids SET pin_sources_incomplete = FALSE - WHERE sha256_hex = $1 AND pin_sources_incomplete", - ) - .bind(sha256_hex) - .execute(&mut *tx) - .await?; + // Clears THIS repo's failure only (#173 round 12). A boolean per object meant + // repo C's genuine record wiped the marker repo B's failure set, and the + // resolver then dropped the scan fallback while B's copy was still unrecorded. + sqlx::query("DELETE FROM pin_source_failures WHERE sha256_hex = $1 AND repo_id = $2") + .bind(sha256_hex) + .bind(repo_id) + .execute(&mut *tx) + .await?; } tx.commit().await?; Ok(()) } - /// Mark this object's pin-source set as KNOWN INCOMPLETE (U3, #173). Called when a - /// `record_pin_source` exhausts its retries, which is the only moment the node - /// knows a source it meant to record is missing. `GET /ipfs/{cid}` reads it to keep - /// the bounded scan fallback for that object, so a public copy that would serve is - /// no longer 404'd. A no-op when no `pinned_cids` row exists (the first-pin path is - /// transactional, so there is no half-recorded pin to describe). - pub async fn mark_pin_sources_incomplete(&self, sha256_hex: &str) -> Result<()> { - sqlx::query("UPDATE pinned_cids SET pin_sources_incomplete = TRUE WHERE sha256_hex = $1") - .bind(sha256_hex) - .execute(&self.pool) - .await?; + /// Mark this object's pin-source set as KNOWN INCOMPLETE for `repo_id` (U3, #173). + /// Called when a `record_pin_source` exhausts its retries, which is the only moment + /// the node knows a source it meant to record is missing. `GET /ipfs/{cid}` reads it + /// to keep the bounded scan fallback for that object, so a public copy that would + /// serve is no longer 404'd. + /// + /// The marker names the PAIR, so only a later successful record from the same repo + /// clears it (#173 round 12). A no-op when no `pinned_cids` row exists: the first-pin + /// path is transactional, so there is no half-recorded pin to describe, and without + /// the guard a marker for an object this node never pinned would sit in the table + /// arming a fallback for nothing. + pub async fn mark_pin_sources_incomplete(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + sqlx::query( + "INSERT INTO pin_source_failures (sha256_hex, repo_id) + SELECT $1, $2 WHERE EXISTS (SELECT 1 FROM pinned_cids WHERE sha256_hex = $1) + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo_id) + .execute(&self.pool) + .await?; Ok(()) } /// Whether this object's pin-source set is KNOWN INCOMPLETE (U3, #173): a - /// `record_pin_source` for it failed outright and no later record has repaired the - /// set. `false` for an unpinned oid and for every row predating the column, so the - /// common path is unchanged and an ordinary denial never fans out (INV-10). + /// `record_pin_source` for it failed outright and no later record from the same repo + /// has repaired it. `false` for an unpinned oid and for every object with no recorded + /// failure, so the common path is unchanged and an ordinary denial never fans out + /// (INV-10). pub async fn pin_sources_incomplete(&self, sha256_hex: &str) -> Result { - let flag: Option = sqlx::query_scalar( - "SELECT pin_sources_incomplete FROM pinned_cids WHERE sha256_hex = $1", - ) - .bind(sha256_hex) - .fetch_optional(&self.pool) - .await?; - Ok(flag.unwrap_or(false)) + let found: Option = + sqlx::query_scalar("SELECT 1 FROM pin_source_failures WHERE sha256_hex = $1 LIMIT 1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(found.is_some()) } /// Every source repository recorded for a pinned object (F1, #173 jatmn round 8): diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 77d0155e..6e3b9cd3 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -934,7 +934,7 @@ pub async fn pin_new_objects( ); if let Err(e) = db_bounded( db_record_deadline(deadline), - db.mark_pin_sources_incomplete(&sha), + db.mark_pin_sources_incomplete(&sha, repo_id), ) .await { @@ -955,7 +955,7 @@ pub async fn pin_new_objects( tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); if let Err(e) = db_bounded( db_record_deadline(deadline), - db.mark_pin_sources_incomplete(&sha), + db.mark_pin_sources_incomplete(&sha, repo_id), ) .await { diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 242821f2..14f1d582 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -262,7 +262,7 @@ pub async fn pin_new_objects( ); if let Err(e) = crate::ipfs_pin::db_bounded( crate::ipfs_pin::db_record_deadline(deadline), - db.mark_pin_sources_incomplete(&sha), + db.mark_pin_sources_incomplete(&sha, repo_id), ) .await { @@ -279,7 +279,7 @@ pub async fn pin_new_objects( tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); if let Err(e) = crate::ipfs_pin::db_bounded( crate::ipfs_pin::db_record_deadline(deadline), - db.mark_pin_sources_incomplete(&sha), + db.mark_pin_sources_incomplete(&sha, repo_id), ) .await { @@ -457,7 +457,7 @@ pub async fn pin_new_objects( ); if let Err(e) = crate::ipfs_pin::db_bounded( crate::ipfs_pin::db_record_deadline(deadline), - db.mark_pin_sources_incomplete(&sha), + db.mark_pin_sources_incomplete(&sha, repo_id), ) .await { @@ -468,7 +468,7 @@ pub async fn pin_new_objects( tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); if let Err(e) = crate::ipfs_pin::db_bounded( crate::ipfs_pin::db_record_deadline(deadline), - db.mark_pin_sources_incomplete(&sha), + db.mark_pin_sources_incomplete(&sha, repo_id), ) .await { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 23f522c9..ded6d45f 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -4290,6 +4290,83 @@ mod tests { ); } + /// #173 round 12 (jatmn): the incompleteness marker is per `(object, repo)`, so a + /// record from an UNRELATED repo does not clear a marker a different repo's failed + /// record set. It was one boolean per object, and the resolver reads a cleared marker + /// as "every source is recorded", drops the scan fallback, and 404s an anonymous + /// caller whose only servable copy is the unrecorded public one. + /// + /// Both directions, because the precision is the point: an unrelated repo must NOT + /// clear, and the repo that actually failed MUST clear, or every transient DB blip + /// would strand an object on the scan path forever. + #[sqlx::test] + async fn pin_source_failure_is_cleared_only_by_the_repo_that_failed(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3perrepo"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3perrepo.git"); + let repo_a = seed_repo(&owner_did, "u3perrepo"); + state.db.create_repo(&repo_a).await.expect("seed repo A"); + let repo_b = seed_repo(&owner_did, "u3perrepo-b"); + state.db.create_repo(&repo_b).await.expect("seed repo B"); + let repo_c = seed_repo(&owner_did, "u3perrepo-c"); + state.db.create_repo(&repo_c).await.expect("seed repo C"); + let _ = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo_a.id).await; + + // Repo B's record fails: the object is now known to be missing B as a source. + state + .db + .mark_pin_sources_incomplete(&fx.public_oid, &repo_b.id) + .await + .expect("mark B's failure"); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "B's failed record marks the set incomplete" + ); + + // A genuine record from an UNRELATED repo C. B is still missing. + state + .db + .record_pin_source(&fx.public_oid, &repo_c.id) + .await + .expect("record C"); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "a record from an unrelated repo must not clear a marker another repo set: \ + the resolver would drop the scan fallback while B's copy is still unrecorded" + ); + + // The repo that actually failed lands its record: now the set is complete. + state + .db + .record_pin_source(&fx.public_oid, &repo_b.id) + .await + .expect("record B"); + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "the repo whose record failed clears its own marker once it lands" + ); + } + /// U3 scenario 5 (#173): the Pinata pin path had BARE `record_pin_source` calls, so /// one transient DB error dropped a source permanently. It now shares the ipfs_pin /// retry helper and marks/clears the same marker. The elapsed-time assertion is the @@ -4975,12 +5052,74 @@ mod tests { ); state .db - .mark_pin_sources_incomplete("preu3oid") + .mark_pin_sources_incomplete("preu3oid", "somerepo") .await .expect("mark after upgrade"); assert!( state.db.pin_sources_incomplete("preu3oid").await.unwrap(), - "the v22 column is present and writable after the upgrade" + "the marker store is present and writable after the upgrade" + ); + } + + /// #173 round 12 (INV-7 upgrade path for v24): a node already carrying v22 markers + /// keeps them across the move to per-`(oid, repo)` state. Which repo failed was never + /// recorded, so a carried marker takes the empty sentinel and no real record clears + /// it, which is strictly safer than the v22 behavior it replaces (there, the next + /// unrelated record cleared it). Also asserts the re-migration is idempotent and that + /// an object with no marker still reads complete. + #[sqlx::test] + async fn pin_source_failures_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v24 shape: drop the new table, forget v24, and leave a v22-style marker. + sqlx::query("DROP TABLE IF EXISTS pin_source_failures") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 24") + .execute(&pool) + .await + .unwrap(); + for (oid, marked) in [("carriedoid", true), ("cleanoid", false)] { + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pin_sources_incomplete) \ + VALUES ($1, $2, $3, $4)", + ) + .bind(oid) + .bind(format!("{oid}cid")) + .bind(chrono::Utc::now().to_rfc3339()) + .bind(marked) + .execute(&pool) + .await + .unwrap(); + } + + state.db.run_migrations().await.expect("re-migrate"); + state + .db + .run_migrations() + .await + .expect("migrations are idempotent: a second run succeeds"); + + assert!( + state.db.pin_sources_incomplete("carriedoid").await.unwrap(), + "a v22 marker survives the upgrade instead of being silently dropped" + ); + assert!( + !state.db.pin_sources_incomplete("cleanoid").await.unwrap(), + "an unmarked row stays complete, so the upgrade arms no new fallback" + ); + + // A real record cannot clear a carried marker: the failing repo is unknown, so + // the sentinel it carries matches no repo id. + state + .db + .record_pin_source("carriedoid", "anyrepo") + .await + .expect("record a source"); + assert!( + state.db.pin_sources_incomplete("carriedoid").await.unwrap(), + "a carried marker names no repo, so nothing clears it by accident" ); } @@ -7429,6 +7568,56 @@ mod tests { // BUDGET, not a per-IP brake: a walk-free public fetch stays un-rate-limited // (ipfs_walk_rate_limited_per_source), while the expensive walk keeps its IP brake. + /// #173 round 12 (jatmn): a failure of the SIZE stage must reach the client as the + /// retryable 503, never as a definitive 404. `object_size_bounded` mapped every + /// non-timeout failure to `Ok(None)`, which `gate_and_serve` read as a verified + /// absence and did not taint the search for, so a corrupt object or a failed spawn + /// 404'd an authorized caller on an object the type probe had just reported present. + /// + /// The failure is induced between the two stages with a test seam, because the size + /// read uses the real `git` rather than `state.git_bin` and no shim can be injected + /// there. The BEFORE request is what makes the AFTER assertion mean anything: it + /// proves this fixture serves 200 when the size read succeeds, so the 503 is caused + /// by the broken size probe and nothing else. + #[sqlx::test] + async fn ipfs_cid_size_probe_failure_is_retryable_not_a_404(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["sizefault"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("sizefault.git"); + let repo = seed_repo(&owner_did, "sizefault"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + let (before, body) = + cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + before, + StatusCode::OK, + "the fixture serves this object when the size read succeeds" + ); + assert!(body.contains("public bytes"), "and serves the real content"); + + // The object vanishes between the type probe and the size probe. Armed on THIS + // repo's path: fixture oids are shared across tests, so an oid-only key would + // reach into another test's repo. + crate::api::ipfs::break_size_probe_for(&bare, &fx.public_oid); + let (after, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + after, + StatusCode::SERVICE_UNAVAILABLE, + "a size-stage fault taints the search and tails to a retryable 503; a 404 here \ + would tell an authorized caller the object does not exist" + ); + } + /// T1 (F1): the probe budget gates BEFORE `acquire`/`cat-file`, so it genuinely /// bounds the fan-out — a repo past the budget is never probed, even one that /// WOULD serve. With the budget at 0, a PUBLIC legacy copy that would otherwise From 9758bc73035fc02d449327c1044bc264faf4ff60 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:23:08 -0500 Subject: [PATCH 40/77] docs(node): name the knob that actually sets the legacy-probe budget The comment beside ipfs_max_legacy_probes told operators to tune GITLAWB_IPFS_MAX_REPOS_WALKED, but AppState::ipfs_legacy_probe_budget reads GITLAWB_IPFS_MAX_LEGACY_PROBES. The knob it named is the separate cap on expensive visibility walks, so anyone following the guidance next to this line adjusted the walk cap and left the legacy scan fan-out at its default. --- crates/gitlawb-node/src/main.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index bd1233f8..4c7a3452 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -421,9 +421,13 @@ async fn main() -> Result<()> { create_ip_rate_limiter, push_rate_limiter, ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, - // The legacy-probe budget is operator-tunable via GITLAWB_IPFS_MAX_REPOS_WALKED - // (R5); the history-walk ceiling above stays constant (a smaller value false-503s - // a provenanced request). Default 256 preserves the shipped behaviour. + // The legacy-probe budget is operator-tunable via GITLAWB_IPFS_MAX_LEGACY_PROBES + // (R5), which is what `ipfs_legacy_probe_budget` reads. NOT + // GITLAWB_IPFS_MAX_REPOS_WALKED, which this comment used to name: that is the + // separate cap on expensive visibility walks, so an operator following the old + // text tuned the walk cap and left this fan-out unchanged. The history-walk + // ceiling above stays constant (a smaller value false-503s a provenanced + // request). Default 256 preserves the shipped behaviour. ipfs_max_legacy_probes: AppState::ipfs_legacy_probe_budget(&config), ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust, From 4985e38af2bd629fa81ec70aec63937ed9894b73 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:16:02 -0500 Subject: [PATCH 41/77] fix(ipfs): page the legacy CID scan instead of materializing every repo GITLAWB_IPFS_MAX_LEGACY_PROBES bounded the probe fan-out, but nothing bounded the load that ran before the first probe. On a legacy (NULL-provenance) or capped/incomplete provenance set, get_by_cid called list_all_repos, cloned every id, fetched visibility rules for the whole set, and fetched the node's entire quarantine list, then retained all three for the request. /ipfs/{cid} is anon-reachable and CIDs are enumerable from the unsigned pins index, and an object pinned from MAX_PIN_SOURCES repos is permanently at cap, so every later anonymous GET for it paid that whole preload with the scarce walk permits held, even at a probe budget of 1. The selection is now keyset-paged on (created_at, id) ASC, with the quarantine flag returned per row so the separate whole-node quarantine query is gone. Both key columns are immutable, so paging is exact: a repo touched mid-scan cannot cross a page boundary and go unvisited, and updated_at is attacker-bumpable, which would have let a caller sort their own repos ahead of the true holder. The pager lives outside the oid loop, so its cursor and its accounting are per request rather than per candidate. Before fetching another page the scan checks that a row on it could still reach a verdict; a spent probe or visit budget stops the read and TAINTS, so a truncated search still sheds the retryable 503 and never a false 404. A page of pure denials is deliberately not a stop condition, since a cheap denial costs no probe and a public object buried behind private repos must still be reached. Both page queries stay clamped to the remaining request budget, and the rules query still fails closed. A new v25 migration adds idx_repos_created_at_id. Bounding what the handler materializes is only half the property: repos carried no index in the new keyset order, so Postgres seq-scanned the table and top-N sorted it to return every page while the walk admission was held, which moves the same O(rows) cost onto the database rather than removing it. Measured on 50k rows: 951 shared buffers and ~44ms per page without the index, versus an Index Only Scan at 7 buffers and ~0.09ms with it, and the keyset predicate pushed down as an Index Cond instead of filtering after a scan. Nothing names the index in any query text, so its entry says why it exists and a test asserts its presence structurally. Two order-dependent tests were reseeded rather than left green: the buried-row and acquire-taint cases encoded their ordering through updated_at, which no longer determines iteration. Both were re-confirmed RED under their documented mutations after the reseed. Three regressions land with the fix. One drives a one-probe, one-row-per-page request against three candidate repos and asserts it materializes one row and runs one query, which a query counter alone could not have caught. One seeds two distinct source-less oids under a single CID and pins the per-request property directly: the second candidate re-reads the pages the first paid for, so the whole request costs one pass rather than one per candidate. The third covers the index, including the INV-7 upgrade path. Refs Gitlawb/node#173. --- crates/gitlawb-node/src/api/ipfs.rs | 606 +++++++++++++++++------ crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/db/mod.rs | 185 ++++++- crates/gitlawb-node/src/main.rs | 1 + crates/gitlawb-node/src/state.rs | 7 + crates/gitlawb-node/src/test_support.rs | 46 +- crates/gitlawb-node/tests/inv22_gates.rs | 10 +- 7 files changed, 659 insertions(+), 197 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f9b84171..f9d9b855 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -74,6 +74,18 @@ pub(crate) const MAX_HISTORY_WALKS_PER_REQUEST: u32 = crate::db::MAX_PIN_SOURCES /// transitional path, not the steady state. Tunable via `AppState`. pub(crate) const MAX_LEGACY_PROBES_PER_REQUEST: u32 = 256; +/// How many repo rows the legacy scan pulls from the database per keyset page +/// (#173, jatmn, INV-10). The probe ceiling above bounds the EXPENSIVE work, but +/// it only starts counting once a probe runs; before that, loading the node's whole +/// repo inventory and every matching visibility rule is itself work proportional to +/// the node's size, bought by one anonymous GET while the scarce walk permits are +/// held. Paging makes the database-facing selection bounded too: the scan reads one +/// page, gates it, and asks for another only while its probe and visit budgets have +/// room. Not an operator knob — sized so a full default-budget scan (256 probes) +/// costs two pages, and a field on `AppState` for the same test-seam reason as the +/// sibling caps. +pub(crate) const LEGACY_SCAN_PAGE_ROWS: usize = 128; + /// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` buffers and serves /// (#173 round 8, F6, INV-10). The serve reads via a blocking `git cat-file` and /// buffers the whole object; unbounded, a large public blob (enumerable from the pins @@ -85,14 +97,114 @@ pub(crate) const MAX_LEGACY_PROBES_PER_REQUEST: u32 = 256; /// `AppState` for the test seam, like the sibling caps. pub(crate) const MAX_SERVED_OBJECT_BYTES: u64 = 32 * 1024 * 1024; -/// Lazily-loaded context for the legacy (NULL-provenance) scan fallback in -/// `get_by_cid`: all repos, their visibility rules keyed by repo id, and the set of -/// quarantined repo ids. Loaded once per request only if a legacy pin is hit. -type LegacyScanCtx = ( - Vec, - HashMap>, - HashSet, -); +/// Keyset pager for the legacy (NULL-provenance) scan fallback in `get_by_cid`. +/// +/// Replaces the old "load every repo, every matching rule, and the whole node's +/// quarantine set up front" preload (#173, jatmn, INV-10). That preload ran before +/// the probe ceiling had spent a single probe, so an anonymous GET for a CID +/// enumerable from the public pins index bought allocation and queries proportional +/// to the node's entire repo and rule inventory, with the scarce walk permits held +/// throughout. Here the scan reads one bounded page at a time and asks for another +/// only while its probe and visit budgets have room. +/// +/// Per REQUEST, not per oid candidate. `get_by_cid` may try several oids under one +/// CID, and a pager that reset between them would restore the full fan-out; instead +/// the cursor, the fetched rows, and their rules persist across the whole request, +/// so a later candidate re-reads the pages already paid for and only ever extends +/// the cursor forward. +#[derive(Default)] +struct LegacyScanPager { + /// Rows fetched so far this request, in `(created_at, id)` ASC order. Bounded by + /// the budgets that gate the next fetch, never by the node's repo count. + rows: Vec, + /// Visibility rules for the fetched rows only, keyed by repo id. + rules: HashMap>, + /// Keyset cursor: the `(created_at, id)` of the last row fetched, `None` before + /// the first page. Both halves are immutable columns, so paging is exact. + cursor: Option<(String, String)>, + /// Set once a short page proves no rows remain after the cursor. + exhausted: bool, +} + +impl LegacyScanPager { + /// Fetch the next page and its rules, appending both. + /// + /// INV-22: these awaits happen while the scarce walk admission is held and the + /// pool sets no `statement_timeout`, so each is clamped to the remaining request + /// budget exactly as the old preload's queries were. A timeout on the rules query + /// FAILS CLOSED — it returns the retryable budget 503 rather than letting the scan + /// continue against an empty rule map and serve a path-scoped object to a caller + /// the rules would have denied. + async fn fetch_next_page( + &mut self, + state: &AppState, + request_deadline: std::time::Instant, + cid_str: &str, + ) -> Result<()> { + #[cfg(test)] + bump_preload_queries(); + let budget_secs = state.config.ipfs_request_budget_secs; + let budget_shed = || { + AppError::Overloaded(format!( + "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" + )) + }; + let after = self + .cursor + .as_ref() + .map(|(created_at, id)| (created_at.as_str(), id.as_str())); + let page = match tokio::time::timeout( + request_deadline.saturating_duration_since(std::time::Instant::now()), + state + .db + .list_repos_page_for_scan(after, state.ipfs_legacy_scan_page_rows as i64), + ) + .await + { + Ok(Ok(page)) => page, + Ok(Err(e)) => return Err(e.into()), + Err(_elapsed) => { + tracing::warn!( + budget_secs, + "/ipfs list_repos_page_for_scan exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; + #[cfg(test)] + note_scan_rows(page.len()); + if page.len() < state.ipfs_legacy_scan_page_rows { + self.exhausted = true; + } + if page.is_empty() { + return Ok(()); + } + let last = page.last().expect("non-empty page has a last row"); + self.cursor = Some((last.created_at_key.clone(), last.repo.id.clone())); + let repo_ids: Vec = page.iter().map(|r| r.repo.id.clone()).collect(); + let rules = match tokio::time::timeout( + request_deadline.saturating_duration_since(std::time::Instant::now()), + state.db.list_visibility_rules_for_repos(&repo_ids), + ) + .await + { + Ok(Ok(rules)) => rules, + Ok(Err(e)) => return Err(e.into()), + Err(_elapsed) => { + tracing::warn!( + budget_secs, + "/ipfs list_visibility_rules_for_repos exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); denying (fail closed) and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; + self.rules.extend(rules); + self.rows.extend(page); + Ok(()) + } +} /// GET /ipfs/{cid} /// @@ -339,10 +451,12 @@ pub async fn get_by_cid( admission: &admission, }; - // Legacy scan context (repos + rules + quarantined ids), loaded LAZILY only when a - // legacy NULL-provenance pin is hit — the provenance path must never trigger the - // O(repos) load (that fan-out is exactly what provenance removes, #173 round 2). - let mut scan_ctx: Option = None; + // Legacy scan pager, advanced LAZILY only when a legacy NULL-provenance pin is hit + // — the provenance path must never trigger it (that fan-out is exactly what + // provenance removes, #173 round 2). Declared here, outside the oid loop, so its + // cursor and its fetched rows are accounted per REQUEST: a per-candidate pager + // would restore the very fan-out the paging removes. + let mut pager = LegacyScanPager::default(); for sha256_hex in &oids { // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). @@ -549,91 +663,61 @@ pub async fn get_by_cid( } }; if needs_scan { - // Load the scan context once, lazily (shared across oid candidates). - if scan_ctx.is_none() { - #[cfg(test)] - bump_preload_queries(); - // F6/KTD-5 (#174): the preload queries run while the scarce walk permits - // are ALREADY held, and the pool sets no statement_timeout, so a query - // blocked in Postgres would pin those slots for the whole stall — past the - // request budget — capacity-503'ing later requests. Clamp each to the - // remaining budget; a timeout returns the same retryable budget 503 the - // later stages shed, and returning here drops the permits. - // `list_visibility_rules_for_repos` is the access-control query, so its - // timeout returns BEFORE the loop: the scan can never run with an empty - // rule map and serve an unfiltered listing that exposes private repos - // (FAIL CLOSED). - let budget_secs = state.config.ipfs_request_budget_secs; - let budget_shed = || { - AppError::Overloaded(format!( - "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" - )) - }; - let repos = match tokio::time::timeout( - request_deadline.saturating_duration_since(std::time::Instant::now()), - state.db.list_all_repos(), - ) - .await - { - Ok(Ok(repos)) => repos, - Ok(Err(e)) => return Err(e.into()), - Err(_elapsed) => { - tracing::warn!( - budget_secs, - "/ipfs list_all_repos exceeded the request budget \ - (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" - ); - return Err(budget_shed()); + // Walk the candidate repos one bounded page at a time. Pages already + // fetched by an earlier oid candidate are re-read from `pager.rows` for + // free; only the tail of the scan costs another query. + let mut idx = 0usize; + loop { + if idx == pager.rows.len() { + if pager.exhausted { + break; } - }; - let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); - let rules_by_repo = match tokio::time::timeout( - request_deadline.saturating_duration_since(std::time::Instant::now()), - state.db.list_visibility_rules_for_repos(&repo_ids), - ) - .await - { - Ok(Ok(rules)) => rules, - Ok(Err(e)) => return Err(e.into()), - Err(_elapsed) => { - tracing::warn!( - budget_secs, - "/ipfs list_visibility_rules_for_repos exceeded the request budget \ - (GITLAWB_IPFS_REQUEST_BUDGET_SECS); denying (fail closed) and freeing the walk permit" - ); - return Err(budget_shed()); + // Buying another page is only worth its query if a row on it could + // still reach a verdict, and a verdict needs the probe and the + // acquire these ceilings are refusing. Spent means stop reading — + // this is the check that keeps the DB-facing selection bounded, so + // a one-probe request cannot pull the node's whole inventory. + // + // Note what is deliberately NOT a stop condition: a page of pure + // denials. A quarantined row or a visibility deny costs no probe, + // so paging must continue past them or a public object buried + // behind many private repos would falsely 404. The budgets above, + // not a page count, are what bound that case. + // + // Stopping here leaves every unread repo unproven, so it TAINTS: + // the tail sheds a retryable 503 naming the ceiling, never a + // definitive 404 (#173, F2). + if walk.probes >= state.ipfs_max_legacy_probes { + walk.taint("probe-ceiling"); + break; } - }; - let quarantined: HashSet = match tokio::time::timeout( - request_deadline.saturating_duration_since(std::time::Instant::now()), - state.db.list_quarantined_repos(), - ) - .await - { - // The quarantine set is also access control (INV-11), so a timeout - // must deny rather than scan with an empty set. - Ok(Ok(rows)) => rows.into_iter().map(|r| r.id).collect(), - Ok(Err(e)) => return Err(e.into()), - Err(_elapsed) => { - tracing::warn!( - budget_secs, - "/ipfs list_quarantined_repos exceeded the request budget \ - (GITLAWB_IPFS_REQUEST_BUDGET_SECS); denying (fail closed) and freeing the walk permit" - ); - return Err(budget_shed()); + if walk.visits >= state.config.ipfs_max_repo_visits { + walk.taint("visit-ceiling"); + break; } - }; - scan_ctx = Some((repos, rules_by_repo, quarantined)); - } - let (repos, rules_by_repo, quarantined) = scan_ctx.as_ref().unwrap(); - for repo in repos { - let rules = rules_by_repo - .get(&repo.id) + pager + .fetch_next_page(&state, request_deadline, &cid_str) + .await?; + if idx == pager.rows.len() { + break; + } + } + let row = &pager.rows[idx]; + idx += 1; + let rules = pager + .rules + .get(&row.repo.id) .map(Vec::as_slice) .unwrap_or(&[]); - let is_quar = quarantined.contains(&repo.id); match gate_and_serve( - &state, repo, rules, is_quar, sha256_hex, &rctx, &mut walk, true, + &state, + &row.repo, + rules, + row.quarantined, + sha256_hex, + &rctx, + &mut walk, + true, ) .await { @@ -1389,6 +1473,31 @@ fn bump_preload_queries() { PRELOAD_QUERIES.with(|c| c.set(c.get() + 1)); } +// Test-only INV-10 cost counter (#173, jatmn): how many repo ROWS the legacy scan's +// database-facing selection actually materialized this request. The query counter above +// cannot see the failure it guards — one unbounded `SELECT ... FROM repos` is a single +// query that pulls the node's entire inventory, so it reads 1 either way. Counting rows +// is what goes red if the paging is reverted. +#[cfg(test)] +thread_local! { + static SCAN_ROWS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_scan_rows() { + SCAN_ROWS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn scan_rows() -> usize { + SCAN_ROWS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_scan_rows(n: usize) { + SCAN_ROWS.with(|c| c.set(c.get() + n)); +} + // Test-only cost counter (F5, #173 round 11): how many times the fallback gate ran the // `pin_sources_at_cap` / `pin_sources_incomplete` pair. The work-budget peek sits ahead // of them, so an already-throttled caller leaves this at 0; putting the peek back after @@ -1540,8 +1649,8 @@ mod closed_pool_tests { ); } - /// #251 / CodeRabbit nit: cover `get_by_cid`'s `list_all_repos` conversion - /// path — a valid CID must still yield 503 on a closed pool. + /// #251 / CodeRabbit nit: cover `get_by_cid`'s DB-error conversion path — a + /// valid CID must still yield 503 on a closed pool. #[sqlx::test] async fn get_by_cid_closed_pool_returns_503_db_unavailable(pool: PgPool) { let state = crate::test_support::test_state(pool.clone()).await; @@ -1795,12 +1904,176 @@ mod tests { git_path.to_str().unwrap().to_string() } + /// #173 (jatmn, INV-10): the legacy scan's DATABASE-facing selection is bounded + /// too, not just its probes. The probe ceiling only starts counting once a probe + /// runs, so before this fix an anonymous GET for a CID enumerable from the public + /// pins index loaded every repo row, every matching visibility rule, and the whole + /// node's quarantine set — work proportional to the node's inventory, bought at a + /// probe budget of 1, with the scarce walk permits held throughout. + /// + /// Page size 1 and probe budget 1 against THREE candidate repos: the scan may read + /// exactly one page, spend its one probe, and stop. It must then report the + /// truncation (503), never a false 404 — the two later repos were never looked at. + /// + /// The ROW count is the load-bearing assertion. A query counter cannot see this + /// regression: reverting to one unbounded `SELECT ... FROM repos` is a single query + /// that pulls the entire inventory, so the query count reads 1 either way. + /// MUTATION (RED): drop the pre-fetch probe-budget check and the pager walks every + /// page anyway — 3 rows materialized and 4 queries instead of 1 and 1. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_legacy_scan_stops_paging_when_the_probe_budget_is_spent( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // One row per page and one probe per request: the smallest configuration in + // which "stopped early" and "read everything" are distinguishable. + state.ipfs_legacy_scan_page_rows = 1; + state.ipfs_max_legacy_probes = 1; + + for name in ["one", "two", "three"] { + seed_repo_with_blob( + &state, + tmp.path(), + "z6pager", + name, + format!("pager row {name}\n").as_bytes(), + ) + .await; + } + + // An oid no repo carries, so every probe reaches a clean absent verdict and the + // only thing that can cut the scan short is the budget under test. + let cid = seed_legacy_pin(&state, &absent_oid()).await; + crate::api::ipfs::reset_scan_rows(); + crate::api::ipfs::reset_preload_queries(); + let peer: SocketAddr = "203.0.113.90:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "stopping early must taint the scan: a truncated search is a retryable 503, \ + never a definitive 404" + ); + assert_eq!( + crate::api::ipfs::scan_rows(), + 1, + "the selection must materialize only the page it can afford to gate, never \ + the node's whole repo inventory (INV-10)" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 1, + "and it must stop asking for pages once the probe budget is spent" + ); + } + + /// #173 (jatmn, INV-10): the pager is per REQUEST, not per oid candidate. + /// + /// The `pinned_cids` index is unique on the git oid but NOT on the cid, so one CID + /// can resolve to several oids and `get_by_cid` tries each. If the pager were + /// re-created inside that loop, every extra candidate would re-page the whole + /// inventory and the fan-out this fix removes would come straight back — a CID with + /// k source-less candidates would cost k full scans of the node. + /// + /// Two DISTINCT absent oids seeded under ONE cid, both source-less so both reach + /// `needs_scan`. Three candidate repos at one row per page, with the probe and visit + /// budgets left at their generous defaults so nothing truncates: the scan runs to + /// exhaustion and 404s honestly. The whole request must cost ONE pass — 4 page + /// queries (3 full pages plus the short page that proves exhaustion) and 3 rows — + /// because the second candidate re-reads rows the first already paid for. + /// + /// MUTATION (RED): shadow `pager` with a fresh `LegacyScanPager::default()` inside + /// the `for sha256_hex in &oids` loop and the counters double to 8 and 6. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_second_oid_candidate_reuses_pages_from_the_first(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // One row per page so each page is individually visible in the counters. The + // probe and visit budgets stay at their defaults: this is about page REUSE, so + // nothing may truncate. + state.ipfs_legacy_scan_page_rows = 1; + + for name in ["one", "two", "three"] { + seed_repo_with_blob( + &state, + tmp.path(), + "z6reuse", + name, + format!("reuse row {name}\n").as_bytes(), + ) + .await; + } + + // Two oids no repo carries, sharing one cid: every probe reaches a clean absent + // verdict, so the scan completes for both candidates and nothing taints. + let first_oid = absent_oid(); + let second_oid = "f3".repeat(32); + let cid = seed_legacy_pin(&state, &first_oid).await; + state + .db + .record_pinned_cid(&second_oid, &cid, None) + .await + .expect("co-locate a second source-less oid under the same cid"); + assert_eq!( + state.db.oids_for_cid(&cid).await.unwrap().len(), + 2, + "precondition: the CID must resolve to two candidates, or the reuse this \ + test is about never happens" + ); + + crate::api::ipfs::reset_scan_rows(); + crate::api::ipfs::reset_preload_queries(); + let peer: SocketAddr = "203.0.113.92:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::NOT_FOUND, + "every candidate reached a verdict under generous budgets, so the honest \ + answer is the definitive 404 — a 503 here would mean something truncated \ + and the counters below would be measuring the wrong thing" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 4, + "the pager is per REQUEST: a second oid candidate re-reads the pages the \ + first already paid for and must never re-query. Expected one pass over 3 \ + repos at 1 row per page = 4 page queries (3 full + the short page that \ + proves exhaustion); a per-candidate pager reads 8" + ); + assert_eq!( + crate::api::ipfs::scan_rows(), + 3, + "and one pass materializes each repo row exactly once (3), not once per \ + oid candidate (6)" + ); + } + /// F2 buried-row repro: with more readable repos than `ipfs_max_repos_walked`, /// existing PUBLIC content past the cap must still serve. The cap counts /// EXPENSIVE walks only — this request has no path-scoped rules anywhere, so it /// runs ZERO walks (the fake-git walk log stays empty) and the cap can never cut - /// the scan: the blob buried in the OLDER-updated repo (iterated last under - /// `list_all_repos`' updated_at DESC) serves its 200. Before F2 the cap counted + /// the scan: the blob buried in the LAST-iterated repo serves its 200. Iteration + /// is `(created_at, id)` ASC since the scan was paged (#173, jatmn), so the + /// blob-carrying repo is seeded LAST to keep it buried. Before F2 the cap counted /// visibility-passing VISITS and broke the loop into the opaque 404 — existing /// content misreported absent because of unrelated repos. MUTATION (RED): count /// visits against the cap again (re-add the check+increment at the visibility @@ -1821,22 +2094,23 @@ mod tests { cfg.ipfs_max_repos_walked = 1; state.config = Arc::new(cfg); - // Seed the blob-carrying repo FIRST so its updated_at is OLDER: the empty - // repo is iterated first and the blob row sits past the old visit budget. - let (_, oid) = seed_repo_with_blob( + // Seed the blob-carrying repo LAST so its created_at is NEWEST: under the + // paged `(created_at, id)` ASC order the empty repo is iterated first and the + // blob row sits past the old visit budget. + seed_repo_with_blob( &state, tmp.path(), "z6f2buried", - "buried", - b"buried row proof\n", + "fresh", + b"unrelated content\n", ) .await; - seed_repo_with_blob( + let (_, oid) = seed_repo_with_blob( &state, tmp.path(), "z6f2buried", - "fresh", - b"unrelated content\n", + "buried", + b"buried row proof\n", ) .await; @@ -1867,7 +2141,7 @@ mod tests { /// F2 walk-cap skip-and-continue: exhausting `ipfs_max_repos_walked` skips the /// walk-NEEDING repo without a verdict but keeps the scan alive. Three public - /// repos carry the same blob, newest first: the first (path-scoped) consumes the + /// repos carry the same blob, in iteration order: the first (path-scoped) consumes the /// cap-of-1 walk and denies (empty allowed-set — a verdict); the second /// (path-scoped) needs a walk the cap forbids and is skipped WITHOUT one (taint); /// the third is plain public and serves the 200 from a cheap probe — found beats @@ -1889,15 +2163,16 @@ mod tests { cfg.ipfs_max_repos_walked = 1; state.config = Arc::new(cfg); - // Insert order = oldest first, so iteration (updated_at DESC) is reversed: - // gatedwalk, then gatedskip, then pubcopy. Identical content -> one CID. + // Iteration is `(created_at, id)` ASC since the scan was paged (#173, jatmn), + // so insert order IS iteration order: gatedwalk, then gatedskip, then pubcopy. + // Identical content -> one CID. let content = b"skip and continue proof\n"; - let (_, oid) = - seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "pubcopy", content).await; - let (skip_id, _) = - seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedskip", content).await; let (walk_id, _) = seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedwalk", content).await; + let (skip_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedskip", content).await; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "pubcopy", content).await; for id in [&walk_id, &skip_id] { state .db @@ -1939,9 +2214,9 @@ mod tests { /// F2 visit ceiling: `ipfs_max_repo_visits` bounds the acquire+probe cost class /// (each visit can be a full Tigris archive fetch on a cache miss). Unlike the /// walk cap there is no cheap way to keep scanning, so exhaustion STOPS the scan - /// — and the stop is a truncation, not an absence: with ceiling 1 the newer - /// empty repo consumes the only visit and the blob-carrying older repo is never - /// probed, so the request sheds a retryable 503 + Retry-After, never a false + /// — and the stop is a truncation, not an absence: with ceiling 1 the + /// first-iterated empty repo consumes the only visit and the blob-carrying repo + /// behind it is never probed, so the request sheds a retryable 503 + Retry-After, never a false /// 404. MUTATION (RED): drop the ceiling check and the blob serves (200); drop /// only the taint on the break and the 503 decays to a 404. #[sqlx::test] @@ -1956,8 +2231,10 @@ mod tests { cfg.ipfs_max_repo_visits = 1; state.config = Arc::new(cfg); - // Blob repo first (older, iterated second); empty repo second (newer, - // consumes the single visit). + // Empty repo seeded first, so under the paged `(created_at, id)` ASC order it + // is iterated first and consumes the single visit; the blob repo behind it is + // never probed. + seed_repo_with_blob(&state, tmp.path(), "z6f2visit", "fresh", b"unrelated\n").await; let (_, oid) = seed_repo_with_blob( &state, tmp.path(), @@ -1966,7 +2243,6 @@ mod tests { b"visit ceiling proof\n", ) .await; - seed_repo_with_blob(&state, tmp.path(), "z6f2visit", "fresh", b"unrelated\n").await; let peer: SocketAddr = "203.0.113.62:5000".parse().unwrap(); let cid = seed_legacy_pin_for_oid(&state, &oid).await; @@ -2078,10 +2354,11 @@ mod tests { /// F2 found-beats-taint on the acquire arm: an acquire timeout taints the /// scan but must NOT stop it — the loop `continue`s, and a later repo that - /// genuinely carries the object still serves. The NEWER row (visited first - /// under `list_all_repos`' updated_at DESC) is a Tigris-backed ghost whose - /// acquire stalls against the silent endpoint and times out at 1s; the - /// OLDER row is a plain public repo carrying the blob, reached next and + /// genuinely carries the object still serves. The FIRST-iterated row (the paged + /// scan orders on `(created_at, id)` ASC since #173/jatmn, so it is the row + /// created first) is a Tigris-backed ghost whose acquire stalls against the + /// silent endpoint and times out at 1s; the row behind it is a plain public + /// repo carrying the blob, reached next and /// served from a cheap probe — found beats taint: 200 with the blob bytes, /// never the truncation 503. MUTATION (RED): turn the acquire-timeout arm's /// `continue` into a `break` and the public copy never serves (503). @@ -2092,42 +2369,44 @@ mod tests { std::fs::create_dir_all(&repos_dir).unwrap(); let mut state = crate::test_support::test_state(pool.clone()).await; state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; - // Seed the blob repo through a LOCAL-ONLY store first, so seeding never - // consults the (deliberately unreachable) Tigris endpoint. + // Seed through a LOCAL-ONLY store first, so seeding never consults the + // (deliberately unreachable) Tigris endpoint. The ghost row goes in FIRST: + // it is a bare DB insert, and under the paged `(created_at, id)` ASC order + // the row created first is the row iterated first. state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone()); + state + .db + .upsert_mirror_repo("z6f2acqcont", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); let content = b"acquire taint continue proof\n"; let (_, oid) = seed_repo_with_blob(&state, tmp.path(), "z6f2acqcont", "pubcopy", content).await; // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare - // repo stays a fast local hit) and add a NEWER ghost row with no local - // copy: its acquire consults the silent local endpoint and stalls to the - // 1s timeout (endpoint-pinned test client, no AWS_* env reads). + // repo stays a fast local hit). The ghost has no local copy, so its acquire + // consults the silent local endpoint and stalls to the 1s timeout + // (endpoint-pinned test client, no AWS_* env reads). let endpoint = crate::test_support::silent_http_endpoint().await; let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) .await; state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); - state - .db - .upsert_mirror_repo("z6f2acqcont", "ghost", "/unused-ghost", None, false) - .await - .unwrap(); let mut cfg = (*state.config).clone(); cfg.git_acquire_timeout_secs = 1; state.config = Arc::new(cfg); - // Ordering precondition: the ghost must be iterated FIRST (updated_at - // DESC — it was upserted after the blob repo), otherwise the pubcopy - // would serve before the taint ever fires and the continue-vs-break - // distinction would go untested. + // Ordering precondition: the ghost must be iterated FIRST, otherwise the + // pubcopy would serve before the taint ever fires and the continue-vs-break + // distinction would go untested. Read through the same paged selection the + // scan uses, so the precondition cannot drift from the real order. let order: Vec = state .db - .list_all_repos() + .list_repos_page_for_scan(None, 100) .await .unwrap() .into_iter() - .map(|r| r.name) + .map(|r| r.repo.name) .collect(); let ghost_pos = order.iter().position(|n| n == "ghost").unwrap(); let pub_pos = order.iter().position(|n| n == "pubcopy").unwrap(); @@ -2601,8 +2880,8 @@ mod tests { /// F3 budget expiry mid-loop: one absolute request budget /// (`ipfs_request_budget_secs`) bounds the whole admitted scan; per-repo /// stages may not each draw a fresh timeout past it. Budget 1s, per-iteration - /// acquire timeout 2s; the NEWER row is a Tigris-backed ghost (no local copy, - /// silent local endpoint) whose acquire stalls, the OLDER row is a plain + /// acquire timeout 2s; the FIRST-iterated row is a Tigris-backed ghost (no local + /// copy, silent local endpoint) whose acquire stalls, the row behind it is a plain /// public repo carrying the blob. The ghost's acquire runs clamped to the ~1s /// remainder and times out; at the next repo the budget gate sees zero /// remaining, taints "budget", and STOPS the scan, so the blob repo is never @@ -2619,10 +2898,17 @@ mod tests { std::fs::create_dir_all(&repos_dir).unwrap(); let mut state = crate::test_support::test_state(pool.clone()).await; state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; - // Seed the blob repo through a LOCAL-ONLY store first, so seeding never - // consults the (deliberately unreachable) Tigris endpoint. + // Seed through a LOCAL-ONLY store first, so seeding never consults the + // (deliberately unreachable) Tigris endpoint. The ghost row goes in FIRST: the + // paged scan orders on `(created_at, id)` ASC (#173, jatmn), so the row created + // first is the row iterated first. state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone()); + state + .db + .upsert_mirror_repo("z6f3budget", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); let (_, oid) = seed_repo_with_blob( &state, tmp.path(), @@ -2632,19 +2918,14 @@ mod tests { ) .await; // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare - // repo stays a fast local hit) and add a NEWER ghost row with no local - // copy: its acquire consults the silent local endpoint and stalls past - // the budget (endpoint-pinned test client, no AWS_* env reads). + // repo stays a fast local hit). The ghost has no local copy, so its acquire + // consults the silent local endpoint and stalls past the budget + // (endpoint-pinned test client, no AWS_* env reads). let endpoint = crate::test_support::silent_http_endpoint().await; let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) .await; state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); - state - .db - .upsert_mirror_repo("z6f3budget", "ghost", "/unused-ghost", None, false) - .await - .unwrap(); let mut cfg = (*state.config).clone(); cfg.ipfs_request_budget_secs = 1; cfg.git_acquire_timeout_secs = 2; @@ -2691,7 +2972,7 @@ mod tests { /// the recorded pid is already dead: a tokio abort would have left it /// running), the log shows the walk started but never completed, and the /// request sheds the terminal budget-truncated 503 without ever reaching the - /// OLDER public copy of the same blob (which would have served 200). After + /// public copy of the same blob behind it (which would have served 200). After /// the response the permit is free: the spawn_blocking closure genuinely /// returned. MUTATION (RED): drop the `min` clamp on `walk_timeout` and the /// walk runs its full 8s sleep (elapsed and log-completion assertions fail). @@ -2742,13 +3023,13 @@ mod tests { cfg.ipfs_request_budget_secs = 2; state.config = Arc::new(cfg); - // Older row: a plain public copy of the same blob, which must never be - // reached. Newer row: path-scoped, so its blob costs the clamped walk. + // First-iterated row (seeded first, `(created_at, id)` ASC): path-scoped, so + // its blob costs the clamped walk. Behind it, a plain public copy of the same + // blob which must never be reached. let content = b"budget walk clamp proof\n"; - let (_, oid) = - seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "pubcopy", content).await; - let (walk_id, _) = + let (walk_id, oid) = seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "gated", content).await; + seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "pubcopy", content).await; state .db .set_visibility_rule( @@ -3740,19 +4021,20 @@ mod tests { ); } - /// F6/KTD-5: the two initial metadata queries (`list_all_repos`, + /// F6/KTD-5: the legacy scan's page queries (`list_repos_page_for_scan`, /// `list_visibility_rules_for_repos`) run AFTER the scarce walk permits are /// acquired (held RAII for the whole request) but BEFORE the per-repo loop's /// first budget gate. Pre-fix they were bare awaits with no deadline, so a query /// blocked in Postgres pinned the walk slot for the whole stall, past the request - /// budget. Here we hold an ACCESS EXCLUSIVE lock on `repos` so `list_all_repos` + /// budget. Here we hold an ACCESS EXCLUSIVE lock on `repos` so the page query /// blocks; with the budget clamp the request sheds a retryable budget 503 within /// ~budget and FREES the walk permit, and a follow-up (lock released) is served. /// /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrapping /// timeout fires (RED — "never returned within budget"). After the fix it returns /// the 503 at ~1s and the permit is free again. MUTATION (RED): drop the - /// `tokio::time::timeout` around `list_all_repos` and this hangs past the wrap. + /// `tokio::time::timeout` around `list_repos_page_for_scan` and this hangs past + /// the wrap. #[sqlx::test] async fn get_by_cid_stalled_metadata_query_frees_walk_permit(pool: sqlx::PgPool) { let mut state = crate::test_support::test_state(pool.clone()).await; @@ -3770,8 +4052,8 @@ mod tests { let router = ipfs_router(state); // Hold an ACCESS EXCLUSIVE lock on `repos` on a dedicated pooled connection: - // `list_all_repos`' SELECT needs ACCESS SHARE, which conflicts, so it blocks - // at lock acquisition regardless of row count. + // the page SELECT needs ACCESS SHARE, which conflicts, so it blocks at lock + // acquisition regardless of row count. let mut lock_conn = pool.acquire().await.unwrap(); sqlx::raw_sql("BEGIN; LOCK TABLE repos IN ACCESS EXCLUSIVE MODE;") .execute(&mut *lock_conn) @@ -4010,7 +4292,9 @@ mod tests { /// listing — exposing a public repo's path-restricted blob. Here a PUBLIC repo /// carries the blob under a path-scoped rule that denies anon; `visibility_rules` /// is locked ACCESS EXCLUSIVE so the rule query blocks. The fix returns the budget - /// 503 BEFORE the loop, so the handler NEVER serves (never 200). + /// 503 BEFORE the loop, so the handler NEVER serves (never 200). Since the scan was + /// paged (#173, jatmn) the rules are fetched per PAGE, so this covers the clamp on + /// every page rather than on one whole-inventory load. /// /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrap fires /// (RED). After the fix it sheds the 503 at ~1s. The `assert_ne!(200)` is the @@ -4052,7 +4336,7 @@ mod tests { let cid = seed_legacy_pin_for_oid(&state, &oid).await; let router = ipfs_router(state); - // Lock `visibility_rules` ACCESS EXCLUSIVE: list_all_repos (on `repos`) still + // Lock `visibility_rules` ACCESS EXCLUSIVE: the page query (on `repos`) still // succeeds, but list_visibility_rules_for_repos blocks on the rule query. let mut lock_conn = pool.acquire().await.unwrap(); sqlx::raw_sql("BEGIN; LOCK TABLE visibility_rules IN ACCESS EXCLUSIVE MODE;") diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index e86b5a5f..509c60fe 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -520,6 +520,7 @@ mod tests { ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 0d1c8140..cee541cd 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -23,6 +23,21 @@ pub struct RepoRecord { pub machine_id: Option, } +/// One row of a keyset page from [`Db::list_repos_page_for_scan`]. +/// +/// Carries the row's quarantine flag inline (so the IPFS scan needs no separate +/// whole-node quarantine query) and the RAW stored `created_at` text, which is +/// the first half of the keyset cursor. The raw text is kept because the keyset +/// comparison is a text comparison and re-serializing the parsed `DateTime` is +/// not guaranteed to reproduce the stored bytes — a cursor that differs from the +/// stored value by one character skips or repeats rows. +#[derive(Debug, Clone)] +pub struct ScanRepoRow { + pub repo: RepoRecord, + pub quarantined: bool, + pub created_at_key: String, +} + /// Per-rule replication mode for a visibility rule. /// `A` hides existence entirely (only valid at whole-repo scope `/`). /// `B` keeps object SHAs and the path visible but withholds content @@ -1045,6 +1060,34 @@ const MIGRATIONS: &[Migration] = &[ // markers it still reads. ], }, + Migration { + version: 25, + name: "repos_created_at_id_index", + stmts: &[ + // #173 (jatmn): backs the keyset order of the paged legacy CID scan + // (`list_repos_page_for_scan`, `ORDER BY created_at ASC, id ASC` with a + // `(created_at, id) > (...)` cursor). The scan replaced a whole-table + // preload precisely to stop one anonymous `GET /ipfs/{cid}` from costing + // work proportional to the node's repo inventory (INV-10), and without this + // index that bound is only half real: `repos` carries no index on + // `(created_at, id)`, so Postgres seq-scans the whole table and top-N sorts + // it to return EVERY page, while the scarce IPFS walk admission is held. + // Measured on a 50k-row fixture: 954 shared buffers and ~47ms per page + // without it, versus an Index Only Scan at 4-5 buffers, ~0.08ms, and + // `Heap Fetches: 0` with it — and the keyset predicate is pushed down as an + // `Index Cond` instead of filtering after a scan. + // + // Column order and direction are load-bearing and must match the query + // exactly; `idx_repos_updated_at` (the order the scan used to use) cannot + // serve this one. NOTHING NAMES THIS INDEX IN ANY QUERY TEXT, so a + // grep-driven "unused index" cleanup will not see its consumer: it is + // reachable from an unauthenticated route and dropping it reopens the + // amplification, so treat it as part of the resolver, not as tuning. + // + // NEW versioned migration (never appended to an applied block, INV-7). + "CREATE INDEX IF NOT EXISTS idx_repos_created_at_id ON repos (created_at ASC, id ASC)", + ], + }, ]; /// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). @@ -1231,7 +1274,7 @@ impl Db { /// Fetch a repo by its stable `id`. Used by the `/ipfs/{cid}` provenance path, /// which resolves a pin straight to its ONE source repo (#173) instead of - /// scanning `list_all_repos`. `id` is exact, so unlike `get_repo`'s fuzzy + /// paging the whole repo table. `id` is exact, so unlike `get_repo`'s fuzzy /// owner/name match there is no mirror-vs-canonical disambiguation. pub async fn get_repo_by_id(&self, id: &str) -> Result> { let row = sqlx::query( @@ -1259,21 +1302,68 @@ impl Db { Ok(rows.into_iter().map(row_to_repo).collect()) } - /// Raw list of every repo row — NOT deduped (a mirror row and its canonical - /// row both appear) and without stars. For enumeration callers that must see - /// every physical row (e.g. the IPFS object scan in `api::ipfs`), not for - /// listing surfaces. Listing surfaces dedupe via `list_all_repos_deduped` or - /// `list_all_repos_with_stars` + `dedupe_canonical_repos`. - pub async fn list_all_repos(&self) -> Result> { + /// One keyset page of raw repo rows for the IPFS object scan (`api::ipfs`) — + /// NOT deduped (a mirror row and its canonical row both appear), since that + /// scan must see every physical row. Listing surfaces dedupe via + /// `list_all_repos_deduped` or `list_all_repos_with_stars` + + /// `dedupe_canonical_repos` and must not use this. + /// + /// Paged rather than whole-table because the scan runs on an anonymously + /// reachable route while holding scarce walk admission: materializing the + /// node's entire repo inventory (plus its rules) before the per-probe budget + /// has spent a single probe is an amplification sink (INV-10). The caller + /// stops asking for pages once its budgets are spent. + /// + /// Ordered on `(created_at, id)` ASC, both IMMUTABLE, so keyset paging is + /// exact: no row is visited twice and none is skipped. `updated_at` would be + /// wrong twice over — a repo touched mid-scan can cross a page boundary and go + /// unvisited (a servable public object misreported as a 404), and it is + /// attacker-bumpable, which would let a caller sort their own repos ahead of + /// the true holder and bury it past the probe budget. + /// + /// `after` is the raw `(created_at, id)` of the last row of the previous page, + /// `None` for the first page. It carries the STORED `created_at` text, not a + /// re-serialized `DateTime`: the comparison is a text comparison and a + /// round-trip through `to_rfc3339` is not guaranteed to reproduce the stored + /// bytes. + /// + /// Each row carries its own `quarantined` flag so the scan needs no separate + /// whole-node quarantine query (INV-11's hard drop stays per row). + pub async fn list_repos_page_for_scan( + &self, + after: Option<(&str, &str)>, + limit: i64, + ) -> Result> { + let (after_created, after_id) = match after { + Some((created_at, id)) => (Some(created_at), Some(id)), + None => (None, None), + }; let rows = sqlx::query( "SELECT id, name, owner_did, description, is_public, default_branch, - created_at, updated_at, disk_path, forked_from, machine_id - FROM repos ORDER BY updated_at DESC", + created_at, updated_at, disk_path, forked_from, machine_id, quarantined + FROM repos + WHERE $1::text IS NULL OR (created_at, id) > ($1::text, $2::text) + ORDER BY created_at ASC, id ASC + LIMIT $3", ) + .bind(after_created) + .bind(after_id) + .bind(limit) .fetch_all(&self.pool) .await?; - Ok(rows.into_iter().map(row_to_repo).collect()) + Ok(rows + .into_iter() + .map(|r| { + let quarantined: bool = r.get("quarantined"); + let created_at_key: String = r.get("created_at"); + ScanRepoRow { + quarantined, + created_at_key, + repo: row_to_repo(r), + } + }) + .collect()) } pub async fn list_all_repos_with_stars(&self) -> Result> { @@ -6532,6 +6622,81 @@ mod ref_certificate_tests { ); } + /// #173 (jatmn), INV-7 + INV-10: the paged legacy CID scan orders on + /// `(created_at, id)` ASC, and `repos` had no index in that order — only + /// `idx_repos_updated_at`, which backed the order the paging REPLACED. Without a + /// matching index Postgres seq-scans `repos` and top-N sorts it to return every + /// page (measured: 954 shared buffers, ~47ms per page on 50k rows) while the + /// scarce IPFS walk admission is held, so the application-side bound the paging + /// buys is cancelled by an O(rows) database cost on an anonymously reachable + /// route. With the index each page is an Index Only Scan at 4-5 buffers with the + /// keyset predicate pushed down as an `Index Cond`. + /// + /// PRESENCE is the whole property, so this asserts it structurally rather than by + /// name: some index on `repos` must lead with `created_at` then `id`, in that + /// order and ascending. A rename is fine; a reorder, a direction flip, or a drop + /// is not. Nothing names this index in any query text, so nothing else would + /// notice its removal. + /// + /// Also the INV-7 upgrade path, in the shape of the v18 test above: an existing + /// node past v1 gets the index from its OWN v25 entry, proven by dropping the + /// index plus its `schema_migrations` row and re-running the real migration code. + /// MUTATION (RED): delete the v25 entry from `MIGRATIONS` and the fresh-chain + /// assertion fails. + #[sqlx::test] + async fn v25_repos_created_at_id_index_applies_on_upgrade(pool: PgPool) { + // Structural, not by name: the leading two columns must be `created_at` then + // `id`, ascending (ASC is the default, so it renders with no DESC). + async fn keyset_index_exists(pool: &PgPool) -> bool { + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS( + SELECT 1 + FROM pg_index i + JOIN pg_class t ON t.oid = i.indrelid + WHERE t.relname = 'repos' + AND i.indnatts >= 2 + AND (SELECT a.attname FROM pg_attribute a + WHERE a.attrelid = t.oid AND a.attnum = i.indkey[0]) = 'created_at' + AND (SELECT a.attname FROM pg_attribute a + WHERE a.attrelid = t.oid AND a.attnum = i.indkey[1]) = 'id' + AND pg_get_indexdef(i.indexrelid) NOT LIKE '%DESC%' + )", + ) + .fetch_one(pool) + .await + .unwrap() + } + + let db = Db::for_testing(pool.clone()); + db.run_migrations().await.unwrap(); + assert!( + keyset_index_exists(&pool).await, + "the paged legacy CID scan's ORDER BY created_at ASC, id ASC must be \ + index-backed, or every page seq-scans and sorts the whole repos table \ + while the IPFS walk admission is held (INV-10)" + ); + + // Simulate a node at pre-v25: drop the index and its migration record. + sqlx::query("DROP INDEX IF EXISTS idx_repos_created_at_id") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 25") + .execute(&pool) + .await + .unwrap(); + assert!( + !keyset_index_exists(&pool).await, + "precondition: index and its migration record removed" + ); + + db.run_migrations().await.unwrap(); + assert!( + keyset_index_exists(&pool).await, + "v25 must recreate the keyset index on an upgrading node" + ); + } + /// INV-7: upgrade-path test — seed a database at v9 with duplicate /// ref_certificates, then let the real v10 migration fire via /// run_migrations(). This exercises the migration code path rather than diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 4c7a3452..9d140fbd 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -429,6 +429,7 @@ async fn main() -> Result<()> { // ceiling above stays constant (a smaller value false-503s a provenanced // request). Default 256 preserves the shipped behaviour. ipfs_max_legacy_probes: AppState::ipfs_legacy_probe_budget(&config), + ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust, sync_trigger_rate_limiter, diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index aa548278..2bd6eccb 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -99,6 +99,13 @@ pub struct AppState { /// Bounds the anonymous `acquire` + `cat-file` fan-out across the node (#173, /// INV-10); a field for the same test-seam reason as `ipfs_max_history_walks`. pub ipfs_max_legacy_probes: u32, + /// How many repo rows the CID resolver's legacy scan pulls per keyset page + /// (default `api::ipfs::LEGACY_SCAN_PAGE_ROWS`). Bounds the DATABASE-facing half + /// of the same fan-out `ipfs_max_legacy_probes` bounds on the probe side: without + /// it the scan materialized every repo row and every matching visibility rule + /// before spending a single probe (#173, INV-10). A field for the same test-seam + /// reason as the sibling caps. + pub ipfs_legacy_scan_page_rows: usize, /// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` will buffer and /// serve (default `api::ipfs::MAX_SERVED_OBJECT_BYTES`). The serve reads via a /// blocking `git cat-file` and buffers the whole object; without a bound a large diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index ded6d45f..b352a94e 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -82,6 +82,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), @@ -8380,9 +8381,10 @@ mod tests { .join("withhold.git"); let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; - // Withholding repo, iterated FIRST (later updated_at; list_all_repos is DESC). + // Withholding repo, iterated FIRST: the paged scan orders on the immutable + // `(created_at, id)` ASC, so the OLDER created_at leads (#173, jatmn). let mut withhold = seed_repo(&owner_did, "withhold"); - withhold.updated_at = Utc::now(); + withhold.created_at = Utc::now() - chrono::Duration::seconds(60); state .db .create_repo(&withhold) @@ -8400,9 +8402,9 @@ mod tests { .await .expect("deny rule"); - // Public copy, no rules, iterated AFTER. + // Public copy, no rules, iterated AFTER (newer created_at). let mut pubcopy = seed_repo(&owner_did, "pubcopy"); - pubcopy.updated_at = Utc::now() - chrono::Duration::seconds(60); + pubcopy.created_at = Utc::now(); state.db.create_repo(&pubcopy).await.expect("pubcopy repo"); // anon: denied at the withholding repo (continue), served from the public copy. @@ -9539,11 +9541,11 @@ mod tests { // surface). The reader is allowed under /secret so the walk returns 200. let secret_tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; - // Oldest `updated_at` → `list_all_repos` (ORDER BY updated_at DESC) probes + // NEWEST `created_at` → the paged scan (ORDER BY created_at, id ASC) probes // this serving repo LAST, so a scan deterministically charges the walk-free // `walkpublic` miss first then this serve: exactly 2 probes per scan. let mut walklimit = seed_repo(&owner_did, "walklimit"); - walklimit.updated_at = chrono::Utc::now() - chrono::Duration::seconds(60); + walklimit.created_at = chrono::Utc::now() + chrono::Duration::seconds(60); state.db.create_repo(&walklimit).await.expect("seed repo"); let rec = state .db @@ -9707,9 +9709,9 @@ mod tests { /// probe-throttled repo since #173-F3) must not end the whole request: the scan /// keeps going so a later walk-free copy still serves, and a spent probe budget is /// a clean 429, never a false 404/503. Otherwise a public CID would 404/429 solely - /// because a newer path-scoped duplicate sorts ahead of an older no-rule copy under - /// `updated_at DESC`. Two same-oid legacy copies: a NEWER `/secret`-scoped denier - /// and an OLDER no-rule public copy. + /// because a path-scoped duplicate sorts ahead of a no-rule copy under the scan's + /// `(created_at, id)` ASC order. Two same-oid legacy copies: a `/secret`-scoped + /// denier iterated first and a no-rule public copy behind it. /// /// Two requests from the SAME IP, budget = 2 (one full scan of both copies): /// req1 probes the denier (charged), its allowed-blob walk denies anon → skip and @@ -9747,10 +9749,11 @@ mod tests { .join("scopeddenier.git"); let secret_cid = pin_cid_for(&denier_bare, &fx.secret_oid, &state.db).await; - // Newer denier: public at "/", `/secret/**` Mode B empty readers → an anon - // blob fetch clears "/", runs the allowed-blob walk, is denied → continue. + // First-iterated denier: public at "/", `/secret/**` Mode B empty readers → an + // anon blob fetch clears "/", runs the allowed-blob walk, is denied → continue. + // The paged scan orders on the immutable `(created_at, id)` ASC (#173, jatmn). let mut denier = seed_repo(&owner_did, "scopeddenier"); - denier.updated_at = Utc::now(); + denier.created_at = Utc::now() - chrono::Duration::seconds(60); state.db.create_repo(&denier).await.expect("seed denier"); state .db @@ -9758,9 +9761,9 @@ mod tests { .await .expect("path rule"); - // Older public copy — NO rule → the secret blob serves via the no-walk path. + // Public copy behind it — NO rule → the secret blob serves via the no-walk path. let mut public = seed_repo(&owner_did, "publiccopy"); - public.updated_at = Utc::now() - chrono::Duration::seconds(60); + public.created_at = Utc::now(); state .db .create_repo(&public) @@ -9818,7 +9821,7 @@ mod tests { /// Load-bearing witness (#173, F4): a readable public copy (no path rule → /// served via the no-walk path, exactly like /// `ipfs_cid_served_from_public_copy_when_withheld_elsewhere`) is given the - /// OLDEST `updated_at` so `list_all_repos` (ORDER BY updated_at DESC) iterates it + /// NEWEST `created_at` so the paged scan (ORDER BY created_at, id ASC) iterates it /// LAST. Ahead of it sit `cap + 1` path-scoped deniers, each forcing an /// allowed-blob walk that denies anon. The cap bounds SPAWNED walks to `cap`, but /// hitting it must `continue` (skip only the walk-requiring denier), NOT `break` @@ -9859,25 +9862,26 @@ mod tests { // no-rule public copy — the proven serve path. let secret_cid = pin_cid_for(&readable_bare, &fx.secret_oid, &state.db).await; - // 1) Readable public copy — OLDEST updated_at → iterated LAST. Public with - // NO visibility rule, so the blob serves via the no-walk path. This is - // the copy an uncapped fan-out would eventually reach and serve. + // 1) Readable public copy — NEWEST created_at → iterated LAST under the paged + // `(created_at, id)` ASC order (#173, jatmn). Public with NO visibility + // rule, so the blob serves via the no-walk path. This is the copy an + // uncapped fan-out would eventually reach and serve. let mut readable = seed_repo(&owner_did, "readable"); - readable.updated_at = Utc::now() - chrono::Duration::seconds(60); + readable.created_at = Utc::now() + chrono::Duration::seconds(60); state .db .create_repo(&readable) .await .expect("seed readable copy"); - // 2) cap+1 deniers with NEWER updated_at → iterated before the copy. Public + // 2) cap+1 deniers with OLDER created_at → iterated before the copy. Public // at "/", but a `/secret/**` Mode B rule with an EMPTY reader list, so an // anon blob fetch clears the "/" gate, runs the allowed-blob walk, and is // denied (the secret blob is in no one's set) → continue. Each distinct // repo.id is its own walk (the memo only dedups the same repo). for name in &denier_names { let mut denier = seed_repo(&owner_did, name); - denier.updated_at = Utc::now(); + denier.created_at = Utc::now(); state.db.create_repo(&denier).await.expect("seed denier"); state .db diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 33815022..48381e3d 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -257,10 +257,10 @@ fn f4_release_keeps_conn_owned_until_unlock_resolves() { ); } -/// F6/KTD-5 (initial IPFS metadata queries deadline-wrapped): `get_by_cid` acquires -/// the scarce walk permits (RAII, held for the whole request) BEFORE its two initial -/// metadata queries, and the per-repo loop's first budget gate runs only later. So -/// both `list_all_repos` and `list_visibility_rules_for_repos` must be clamped to the +/// F6/KTD-5 (IPFS metadata queries deadline-wrapped): `get_by_cid` acquires +/// the scarce walk permits (RAII, held for the whole request) BEFORE its metadata +/// queries, and the per-repo loop's first budget gate runs only later. So +/// both `list_repos_page_for_scan` and `list_visibility_rules_for_repos` must be clamped to the /// remaining request budget — otherwise a query blocked in Postgres pins the walk slot /// for the whole stall, past the budget. This scans the PRODUCTION half of `api/ipfs.rs` /// (the `mod tests` half names the same calls in its own harness and would make the @@ -285,7 +285,7 @@ fn f6_ipfs_metadata_queries_are_deadline_wrapped() { // every time a call site is added, which is how a guard quietly stops covering the // site that matters. Requiring all of them scales with the code instead. for call in [ - ".list_all_repos()", + ".list_repos_page_for_scan(", ".list_visibility_rules_for_repos(", ".get_repo_by_id(", ".is_repo_quarantined(", From d88c0c726c8630e104e0161366d1815d69e96428 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 10 Aug 2026 21:52:12 -0500 Subject: [PATCH 42/77] fix(node): repair source-less legacy pins by bounded additive discovery The boot sweep resolved each legacy row's bytes only through pin_sources_for_oid. A row predating provenance has repo_id NULL and no pin_repo_sources entry, so the source set is empty, the `for repo_id in sources` body never runs, and the cursor advances past it. list_pinned_cids also filters those rows out on is_raw_cidv1, so they were both unadvertised and unrepairable: nothing on the node would ever fix them. The sweep exists precisely for pre-provenance rows, and that was the shape it could not handle. For a source-less row, discovery now probes a bounded set of warm local repos for the object and, on a hit, repairs the key through the existing repair_legacy_provider_cid. What it records is the part worth reading closely. It calls record_pin_source additively and then mark_pin_sources_incomplete. It does NOT call backfill_pin_provenance, and repo_id stays NULL. Reading identical bytes out of a repo proves that repo HOLDS the object, not that it is the source: forks, a vendored file, a shared LICENSE blob and the empty tree all collide across repos. Since backfill_pin_provenance guards on AND repo_id IS NULL, the first claim written is permanent, so an exclusive claim would let probe order decide which repo's visibility rules gate a public read surface, with no correction path. It would also disable the resolver's own fallback: needs_scan is sources.is_empty() || at_cap || incomplete, so a non-empty below-cap unmarked set reads as complete and stops the scan that would have found a servable copy. The additive record plus the incomplete marker keeps that fallback alive and makes nothing permanent. Candidates are quarantined-filtered. list_all_repos is a bare SELECT over repos with no quarantine or visibility filter, while the resolver's own legacy scan loads list_quarantined_repos and gates on it, so discovery loads the same set and drops those ids. A quarantined repo is hidden from every reader, so it must not become a recorded source. Private non-quarantined repos stay in the list: an additive source record does not widen what the resolver will serve, since every source is gated independently. The candidate list and the warm filter load once per pass, lazily, on the first source-less row, and the is_dir sweep runs under spawn_blocking rather than inline on a worker. Nothing goes through repo_store.acquire, so a cold repo is never pulled back from remote storage; a repair pass must not become a bulk restore. Two classification rules matter more than they look. A cold or quarantined candidate does NOT mark the row retryable. The candidate list is every repo on the node rather than the row's holders, and sweep_legacy_provider_cids rewinds the cursor whenever any retryable skip occurred, so counting a cold candidate would make every source-less row retryable on every run on a cold-storage node: the cursor would rewind forever, the sweep would never drain, and each pass would pay full discovery cost. Only a read error on a warm candidate is evidence about the row. Conversely, cap exhaustion IS retryable rather than terminal, and candidates are ordered oldest-first by (created_at, id) rather than by id: repo_id derives from the owner DID, which anyone can grind, so a first-N-wins cap over an id-sorted list would let someone bury the true holder permanently behind cheap registrations. MAX_LEGACY_DISCOVERY_PROBES bounds the expensive unit, a bounded object read from a warm repo. A candidate dropped cold, quarantined, or unsafe costs no probe. Its doc states the contract it encodes rather than its resemblance to the resolver's own cap, which happens to share the value under a different contract. The source record is best-effort and says so at the site rather than claiming a later pass heals it. Once the key is rewritten the row is raw CIDv1, so the cost gate skips it free and nothing revisits it; the healing path is the resolver's needs_scan fallback, which the incomplete marker keeps available. The no-remote-fetch property is asserted from both sides. On the effect side, a cold candidate's disk path still does not exist after two full runs, with a control proving the sweep declined an available copy rather than finding nothing to take. On the call side, a source scan of this module (comments and string literals stripped, four anti-vacuity assertions) rejects the fetch-capable call shapes, and a mutation planting a real acquire in the probe loop turns it red. What that pair does not cover, and the test says so: a fetch reached transitively through git::store or db, and git's own promisor lazy-fetch on a partial clone. Neither shape exists in the tree today. Concurrency around the marker is executed rather than assumed. Discovery records then marks, in that order, because a successful record_pin_source clears the marker. Under a proven lock-wait interleaving, a concurrent record committing before discovery's mark leaves the row marked, and two concurrent passes over one row converge on a single source with the marker set. Both the swap and the drop of that pair are red-checked. One residual is pinned by a test rather than papered over, and it is not introduced here. pin_sources_incomplete is one boolean per object, not per (object, repo), so a genuine pusher whose record_pin_source commits strictly AFTER discovery's mark clears it, leaving a non-empty below-cap unmarked set that tells needs_scan to stop scanning. No ordering inside this sweep can prevent that, and record_pin_source's own doc already states the same residual for its existing call sites; closing it needs a per-(oid, repo) marker, which is a change to the marker's shape. It is bounded in practice: the clearing writer really holds the bytes, so every source in the set is servable, and for these rows it cannot be a regression, since before discovery ran they were keyed on a provider CID and were neither served nor advertised. Tests: fifteen regressions driving the sweep entry points. The repair, cheap second pass, probe cap and multi-holder cases were observed RED first. The must-not cases carry mutations instead, since a guard against something happening has no honest pre-fix red: quarantined holder never serves, a cold candidate is neither fetched nor materialized and does not rewind the cursor, an unsafe candidate name is dropped without aborting the pass while a later safe candidate still repairs, a raw-CIDv1 row never enters discovery, and a provenanced row still resolves through the source loop alone. Every guard carries a red-check mutation with an attributed message, including three on the design itself: one adds backfill_pin_provenance beside the additive record and must redden the no-exclusive-claim assertion, one neutralizes the incomplete marker, and one swaps the record and mark order. --- crates/gitlawb-node/src/ipfs_pin.rs | 233 ++++- crates/gitlawb-node/src/test_support.rs | 1214 +++++++++++++++++++++++ 2 files changed, 1446 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 6e3b9cd3..357502c2 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -11,6 +11,22 @@ use anyhow::Result; use gitlawb_core::cid::Cid; use std::time::{Duration, Instant}; +/// How much READ work one source-less legacy row may cost a boot-time sweep. +/// +/// The contract this number encodes: discovery for one pre-provenance row is allowed +/// at most this many bounded object reads from warm local repos, whatever the node's +/// repo count. The unit counted is the expensive one, a `git cat-file` pair against a +/// candidate repo; a candidate rejected at filter time (quarantined, cold, an unsafe +/// path) costs nothing against it. Without the cap the sweep is the O(repos x objects) +/// fan-out this subsystem's cost rule exists to forbid, paid at boot on the node with +/// the most history. +/// +/// It happens to equal the resolver's serve-time per-request source cap +/// (`db::MAX_PIN_SOURCES`), but it is a different bound with a different owner: that +/// one bounds how many sources ONE `/ipfs` request may gate, this one bounds how many +/// repos ONE background row may read. If either moves, the other does not follow. +pub(crate) const MAX_LEGACY_DISCOVERY_PROBES: usize = 16; + /// Attempts (including the first) for a transient DB-record retry. const PIN_RECORD_ATTEMPTS: u32 = 3; /// Backoff between DB-record retry attempts. @@ -293,6 +309,194 @@ pub(crate) struct SweepStats { /// early keeps the cursor, so the next boot resumes past the rows already walked rather /// than repeating them, and the table still gets covered across boots. pub(crate) const MAX_DEAD_ROW_READS_PER_RUN: usize = 64; +/// The warm, non-quarantined repos one pass may probe for a source-less legacy row, +/// plus the one absolute deadline every probe in the pass shares. +/// +/// Loaded LAZILY, once per pass, on the first source-less row, mirroring the resolver's +/// own legacy-scan context: a pass with no such row pays nothing. The `is_dir` warm +/// filter runs ONCE here rather than per row, on the blocking pool, because O(repos) +/// stat calls per row would park a tokio worker for the whole boot sweep. +struct DiscoveryCtx { + /// Warm candidates with their validated disk paths, oldest-first by + /// `(created_at, id)`. + candidates: Vec<(crate::db::RepoRecord, std::path::PathBuf)>, + /// Shared by every discovery read in the pass, so one pass's discovery costs at + /// most one `git_timeout` in total on top of the per-row probe cap. + deadline: Instant, +} + +/// Build one pass's discovery candidate list. +/// +/// Three filters, all applied before any probe so a rejected candidate costs nothing +/// against [`MAX_LEGACY_DISCOVERY_PROBES`]: +/// +/// - QUARANTINE. `list_all_repos` is a bare SELECT with no visibility or quarantine +/// filter, and a quarantined repo is hidden from every reader, so it must not become +/// a discovery source either. The resolver's legacy scan loads the same set and gates +/// on it. Private, non-quarantined repos DO stay in the list: an additive source +/// record binds nothing to one repo's ACL, because the resolver gates every source +/// independently at serve time, so probing a private repo leaks nothing. +/// - WARM ONLY. The path is resolved through the repo store's validated resolver and +/// kept only if it is on local disk. Nothing here goes through `repo_store.acquire`: +/// the sweep is opportunistic background maintenance over every pinned row on the +/// node, and pulling cold repos back from remote storage would turn a repair pass +/// into a bulk restore. +/// - UNSAFE PATH. A name that fails the validated resolver is dropped with a warn and +/// is terminal; nothing a later run changes. +/// +/// The survivors are ordered oldest-first by `(created_at, id)` rather than by id +/// alone. `repo_id` derives from the owner DID, which anyone can grind, so an id sort +/// would let an attacker register low-sorting repos and push the true holder past the +/// probe cap. Source-less rows predate provenance and their holders are old repos, +/// while freshly registered repos sort last and cannot be backdated. +async fn load_discovery_ctx( + repos_dir: &std::path::Path, + git_timeout: Duration, + db: &crate::db::Db, +) -> Result { + let repos = db.list_all_repos().await?; + let quarantined: std::collections::HashSet = db + .list_quarantined_repos() + .await? + .into_iter() + .map(|r| r.id) + .collect(); + let mut candidates: Vec = repos + .into_iter() + .filter(|r| !quarantined.contains(&r.id)) + .collect(); + candidates.sort_by(|a, b| { + a.created_at + .cmp(&b.created_at) + .then_with(|| a.id.cmp(&b.id)) + }); + + let repos_dir = repos_dir.to_path_buf(); + let warm = tokio::task::spawn_blocking(move || { + candidates + .into_iter() + .filter_map(|repo| { + match crate::git::repo_store::validated_repo_disk_path( + &repos_dir, + &repo.owner_did, + &repo.name, + ) { + Ok(p) if p.is_dir() => Some((repo, p)), + // Cold: not on this node's disk right now. It is not evidence about + // any row (see `discover_legacy_row`), so it is simply absent here. + Ok(_) => None, + Err(e) => { + tracing::warn!(repo_id = %repo.id, err = %e, "sweep discovery: rejected unsafe repo path"); + None + } + } + }) + .collect::>() + }) + .await?; + + Ok(DiscoveryCtx { + candidates: warm, + deadline: Instant::now() + git_timeout, + }) +} + +/// What discovery did with one source-less legacy row, in the same three-way shape +/// [`RepairOutcome`] uses so the row accounting is unchanged. +enum DiscoveryOutcome { + /// Nothing here a later run would find either. + Settled, + /// Worth walking again: a warm candidate's read failed, the candidate list could + /// not be loaded, or the probe cap was reached with candidates still unprobed. + Retryable, + /// The row's key was rewritten from bytes verified in a warm local repo. + Repaired, +} + +/// Probe a bounded set of warm local repos for a source-less legacy row's object. +/// +/// On a hit, record ONLY what discovery actually knows. Reading identical bytes proves +/// the repo HOLDS the object, not that it is the first pinner: forks, a shared LICENSE +/// blob and the empty tree all collide, and `backfill_pin_provenance`'s +/// `AND repo_id IS NULL` guard would make a guessed exclusive claim permanent. Worse, +/// the resolver's `needs_scan` is `sources.is_empty() || at_cap || incomplete`, so an +/// exclusive claim would permanently disable the fallback scan for that object. So +/// `pinned_cids.repo_id` stays NULL, the discovered repo goes in ADDITIVELY, and the +/// incomplete marker goes with it because one discovered holder never proves the set +/// complete. +/// +/// Both writes are best-effort and warn-only, and the degradation is stated rather +/// than deferred to a healing pass that does not exist: if the source record fails the +/// row is raw-CIDv1 with an empty or incomplete source set, which is exactly the state +/// `needs_scan` routes to the bounded legacy scan, so the object stays servable. The +/// sweep itself never revisits it (the cost gate skips a raw row free from then on), +/// so the resolver's fallback is the healing path, not a retry. +async fn discover_legacy_row( + sha: &str, + ctx: &mut Option>, + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + db: &crate::db::Db, +) -> DiscoveryOutcome { + if ctx.is_none() { + *ctx = Some(match load_discovery_ctx(repos_dir, git_timeout, db).await { + Ok(c) => Some(c), + Err(e) => { + tracing::warn!(err = %e, "sweep discovery: failed to load the candidate list"); + None + } + }); + } + let ctx = match ctx.as_ref().expect("the candidate list was just loaded") { + Some(c) => c, + // A failed load says nothing about the row, so a later run retries it. + None => return DiscoveryOutcome::Retryable, + }; + + let mut retryable = false; + // Every candidate that gets this far is READ, so taking the first + // MAX_LEGACY_DISCOVERY_PROBES bounds the expensive work exactly. Candidates the + // filters already rejected never reach here and so cost nothing against the cap. + for (repo, repo_path) in ctx.candidates.iter().take(MAX_LEGACY_DISCOVERY_PROBES) { + match repair_legacy_provider_cid(repo_path, git_bin, ctx.deadline, sha, db).await { + Ok(RepairOutcome::Repaired) => { + if let Err(e) = db.record_pin_source(sha, &repo.id).await { + tracing::warn!(sha = %sha, repo_id = %repo.id, err = %e, "sweep discovery: failed to record the discovered pin source"); + } + // After the record, never before: a successful `record_pin_source` + // clears this marker. + if let Err(e) = db.mark_pin_sources_incomplete(sha).await { + tracing::warn!(sha = %sha, err = %e, "sweep discovery: failed to mark the pin-source set incomplete"); + } + return DiscoveryOutcome::Repaired; + } + // The bytes could not be read from this WARM candidate right now, which IS + // evidence about the row: try the next one and walk the row again later. + Ok(RepairOutcome::Retryable) => retryable = true, + // Absent here, or the row was repaired concurrently. Next candidate. + Ok(RepairOutcome::Settled) => {} + Err(e) => { + tracing::warn!(sha = %sha, repo_id = %repo.id, err = %e, "sweep discovery: probe failed"); + retryable = true; + } + } + } + if ctx.candidates.len() > MAX_LEGACY_DISCOVERY_PROBES { + // Cap exhausted with candidates left unprobed: RETRYABLE, never terminal. The + // probe order is deterministic, but "a re-walk finds the same nothing" only + // holds if the candidate set cannot be steered, and it can: repo ids derive + // from grindable owner DIDs, so a terminal verdict would let an attacker bury + // the true holder past the cap permanently. The oldest-first order makes that + // expensive, and this arm makes it non-permanent. + return DiscoveryOutcome::Retryable; + } + if retryable { + DiscoveryOutcome::Retryable + } else { + DiscoveryOutcome::Settled + } +} /// One bounded pass of the U4 sweep: read at most `batch` `pinned_cids` rows after /// the persisted cursor, repair the legacy ones, and persist the new cursor. @@ -319,6 +523,10 @@ async fn sweep_pass( let mut retryable_skips = 0usize; let mut dead_row_reads = 0usize; let mut last = cursor; + // Loaded on the first source-less row and reused by every later one. The outer + // `None` is "not loaded yet"; `Some(None)` is "the load failed this pass", which is + // remembered so a broken DB is not re-queried once per row. + let mut discovery: Option> = None; for (sha, stored) in rows { // Advance FIRST: every path below this line may skip the row, and none of them @@ -331,7 +539,10 @@ async fn sweep_pass( } // Resolve the row's repo from its recorded provenance (first-pinner plus the // bounded additional source set). An empty set is a pin recorded before - // provenance existed: nothing to read the bytes from, so skip it. + // provenance existed, which is the pre-provenance-est shape of row and exactly + // what this sweep is for, so it is not skipped: discovery below probes a + // bounded, quarantine-filtered set of warm local repos for the object and + // records what it finds ADDITIVELY. let sources = match db.pin_sources_for_oid(&sha).await { Ok(s) => s, Err(e) => { @@ -348,6 +559,18 @@ async fn sweep_pass( // Whether any source got as far as spending an object read on this row, which is // what makes an unrepairable row COST something rather than just being skipped. let mut row_read_attempted = false; + if sources.is_empty() { + match discover_legacy_row(&sha, &mut discovery, repos_dir, git_bin, git_timeout, db) + .await + { + DiscoveryOutcome::Repaired => { + repaired += 1; + row_repaired = true; + } + DiscoveryOutcome::Retryable => row_retryable = true, + DiscoveryOutcome::Settled => {} + } + } for repo_id in sources { let repo = match db.get_repo_by_id(&repo_id).await { Ok(Some(r)) => r, @@ -459,6 +682,14 @@ pub(crate) async fn sweep_legacy_provider_cids_once( /// the node already has, so on an upgraded node that push generally never comes. This /// walks the table instead. /// +/// A row with NO recorded source is the pre-provenance case this exists for, so it is +/// not skipped: the pass probes a bounded, quarantine-filtered set of WARM local repos +/// for the object (at most [`MAX_LEGACY_DISCOVERY_PROBES`] reads per row, sharing one +/// per-pass deadline) and, on a hit, rewrites the key from the verified bytes and +/// records the discovered repo ADDITIVELY alongside the incomplete marker. It never +/// writes an exclusive first-pinner claim and never pulls a cold repo back from remote +/// storage. See `discover_legacy_row` for why both of those matter. +/// /// Runs until a pass comes back short of a full batch, which is the end of the table. /// Sleeps `delay` between full batches so it cannot monopolize the DB, and persists /// its cursor every pass so a restart continues instead of rewinding. Errors reading diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index b352a94e..c3f03f20 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -7388,6 +7388,1220 @@ mod tests { ); } + // ---- F1: bounded additive discovery for source-less legacy rows ---- + + /// An empty bare repo at `path`, used as a warm discovery candidate that does not + /// hold the object. sha256 so a 64-hex oid probe is a clean "absent" rather than a + /// format error. + fn init_empty_bare(path: &std::path::Path) { + std::fs::create_dir_all(path.parent().unwrap()).expect("create the owner dir"); + let out = std::process::Command::new("git") + .args([ + "init", + "-q", + "--bare", + "--object-format=sha256", + path.to_str().unwrap(), + ]) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git init --bare: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + + async fn set_quarantined(pool: &PgPool, repo_id: &str) { + sqlx::query("UPDATE repos SET quarantined = TRUE WHERE id = $1") + .bind(repo_id) + .execute(pool) + .await + .unwrap(); + } + + async fn pinned_repo_id(pool: &PgPool, oid: &str) -> Option { + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = $1") + .bind(oid) + .fetch_one(pool) + .await + .unwrap() + } + + /// F1 scenario 1 (#173): a pre-provenance row (`repo_id` NULL, no `pin_repo_sources` + /// entry) is repaired by probing warm local repos for the object, and the discovered + /// repo is recorded ADDITIVELY. The must-not half is the last assertion: reading + /// identical bytes proves the repo HOLDS the object, never that it is the FIRST + /// pinner (forks, a shared LICENSE blob and the empty tree all collide), and + /// `backfill_pin_provenance`'s `AND repo_id IS NULL` guard would make a guessed + /// exclusive claim permanent, so `pinned_cids.repo_id` must stay NULL. RED before + /// discovery exists: the source set is empty, the row is skipped, and the cursor + /// advances past it for good. + #[sqlx::test] + async fn sweep_discovery_repairs_sourceless_legacy_row(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["discsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("discsrc.git"); + let repo = seed_repo(&owner_did, "discsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // The pre-provenance shape: NULL repo_id and no pin_repo_sources row. + let (raw_cid, provider_cid) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + assert!( + state + .db + .pin_sources_for_oid(&fx.public_oid) + .await + .unwrap() + .is_empty(), + "the seeded row really has no recorded source" + ); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates"); + assert_eq!(stats.repaired, 1, "discovery repairs the source-less row"); + + let (stored, stashed) = stored_pin(&pool, &fx.public_oid).await; + assert_eq!( + stored, raw_cid, + "the key is rewritten to the raw-content CID from locally verified bytes" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![repo.id.clone()], + "the discovered repo is recorded as an additive source" + ); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "one discovered holder never proves the set complete, so the marker is set \ + and the resolver's fallback scan stays available" + ); + assert_eq!( + pinned_repo_id(&pool, &fx.public_oid).await, + None, + "discovery makes no exclusive first-pinner claim: repo_id stays NULL" + ); + } + + /// F1 scenario 2 (#173): once discovery has repaired the row it is raw-CIDv1, so a + /// later pass takes the cost gate's cheap path and reads no bytes at all. The cursor + /// is rewound by hand so the second pass really re-walks the row rather than reading + /// nothing because it is behind the cursor. + #[sqlx::test] + async fn sweep_discovery_repaired_row_is_cheap_on_later_passes(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["cheapsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("cheapsrc.git"); + let repo = seed_repo(&owner_did, "cheapsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let (raw_cid, _provider) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + let first = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the first run terminates"); + assert_eq!(first.repaired, 1, "the first run repairs by discovery"); + + // Re-walk the same row: the cost gate must spare it every byte read. + state.db.set_pin_repair_cursor("").await.unwrap(); + crate::ipfs_pin::reset_legacy_repair_reads(); + let second = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the second run terminates"); + assert_eq!(second.scanned, 1, "the second run really re-walks the row"); + assert_eq!(second.repaired, 0, "there is nothing left to repair"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a repaired row is raw-CIDv1, so no later pass reads bytes for it" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the repaired key survives the second pass" + ); + } + + /// F1 scenario 3 (#173, MUST-NOT): a quarantined repo is hidden from every reader, + /// so it must not become a discovery source either. The only holder here is warm and + /// quarantined, and the filter drops it at candidate-load time, before any probe: the + /// row is left exactly as it is and no bytes are read. + #[sqlx::test] + async fn sweep_discovery_skips_quarantined_holder(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["quarsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("quarsrc.git"); + let repo = seed_repo(&owner_did, "quarsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + set_quarantined(&pool, &repo.id).await; + let (_raw_cid, provider_cid) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates"); + + assert_eq!( + stats.repaired, 0, + "a quarantined repo never serves as a discovery source" + ); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "the quarantine filter drops the candidate before any probe reads bytes" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "the row keeps its provider key" + ); + assert_eq!( + state + .db + .pin_sources_for_oid(&fx.public_oid) + .await + .unwrap() + .len(), + 0, + "no source is recorded from a quarantined repo" + ); + } + + /// F1 scenario 4 (#173, MUST-NOT): a candidate that is not on local disk is COLD. + /// Discovery must not pull it back from remote storage (the sweep is opportunistic + /// background maintenance, not a bulk restore), and it must not mark the row + /// retryable either. The candidate list is every repo on the node rather than the + /// row's holders, so on a Tigris-backed node where most repos are cold a + /// cold-candidate retryable would rewind the cursor + /// (`sweep_legacy_provider_cids` rewinds whenever `retryable_skips > 0`) on every + /// run: the sweep would never drain while paying full discovery cost each pass. The + /// second run's `scanned` is what proves the cursor was not rewound. + /// + /// The no-fetch half is asserted here on the EFFECT rather than on the call: the + /// cold candidate's disk path must still not exist after two full runs, which is + /// what any restore (through the repo store, through Tigris, through anything else) + /// would have changed. The control that keeps that assertion from being vacuous is + /// the read from `stashed_away`: the bytes really are still on this node and really + /// would have repaired the row, so declining them is a choice and not an absence. + /// The call-shape half is `sweep_module_never_calls_a_remote_fetch`. + #[sqlx::test] + async fn sweep_discovery_cold_candidates_do_not_rewind(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["coldcand"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("coldcand.git"); + let repo = seed_repo(&owner_did, "coldcand"); + state.db.create_repo(&repo).await.expect("seed repo"); + let (raw_cid, provider_cid) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + // The only holder goes cold: its row stays in the DB, its bytes leave the disk. + let stashed_away = bare.with_extension("git.away"); + let _ = std::fs::remove_dir_all(&stashed_away); + std::fs::rename(&bare, &stashed_away).expect("take the repo off local disk"); + + crate::ipfs_pin::reset_legacy_repair_reads(); + let first = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the first run terminates"); + assert_eq!( + (first.scanned, first.repaired), + (1, 0), + "the row is walked and no cold candidate can repair it" + ); + // The effect a restore would have left behind. Nothing put the repo back. + assert!( + !bare.exists(), + "the sweep must never materialize a cold candidate on local disk: a repair \ + pass over every pinned row on the node would become a bulk restore" + ); + // Anti-vacuity for the assertion above: the bytes are still reachable on this + // node and still recompute to the raw key, so a fetch would have succeeded and + // repaired the row. The sweep declined an available copy rather than finding + // nothing to take. + let (_ty, stashed_bytes) = crate::git::store::read_object(&stashed_away, &fx.public_oid) + .expect("the stashed copy is readable") + .expect("the stashed copy still holds the object"); + assert_eq!( + gitlawb_core::cid::Cid::from_git_object_bytes(&stashed_bytes).to_string(), + raw_cid, + "the withheld copy is exactly the one that would have repaired the row" + ); + assert_eq!( + first.retryable_skips, 0, + "a cold candidate is not evidence about the row, so it never marks the row \ + retryable and never drives a cursor rewind" + ); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a cold candidate is filtered at load, so nothing is read and nothing is pulled" + ); + + let second = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the second run terminates"); + assert_eq!( + (second.scanned, second.retryable_skips), + (0, 0), + "the cursor was not rewound: the second run re-reads nothing" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "the row keeps its provider key" + ); + assert!( + !bare.exists(), + "no later pass materialized the cold candidate either" + ); + } + + /// F1 scenario 5 (#173, BOUND plus anti-burial): the probe cap counts the expensive + /// unit, a bounded object read from a warm repo, so a row costs at most + /// `MAX_LEGACY_DISCOVERY_PROBES` reads however many candidates the node holds. With + /// candidates left over the row is classified RETRYABLE, not terminal: `repo_id` + /// derives from the owner DID, which anyone can grind, so a first-N-wins cap over a + /// sorted set would otherwise let an attacker bury the true holder permanently. + #[sqlx::test] + async fn sweep_discovery_read_probes_are_capped(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + // The bytes live in a bare repo with NO repos row, so it is never a candidate. + let fx = seed_cid_repos(&slug, &short, &["capsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("capsrc.git"); + let (_raw_cid, provider_cid) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + // More warm candidates than the cap, none of them holding the object. + let candidates = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES + 4; + for i in 0..candidates { + let name = format!("capcand{i}"); + init_empty_bare( + &std::path::PathBuf::from("/tmp") + .join(&slug) + .join(format!("{name}.git")), + ); + let repo = seed_repo(&owner_did, &name); + state.db.create_repo(&repo).await.expect("seed candidate"); + } + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = tokio::time::timeout( + std::time::Duration::from_secs(60), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates"); + + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES, + "one row costs at most the probe cap in object reads, whatever the candidate count" + ); + assert_eq!(stats.repaired, 0, "no candidate holds the object"); + assert_eq!( + stats.retryable_skips, 1, + "cap exhaustion with candidates remaining is RETRYABLE, so a buried holder \ + is re-walked by a later run instead of written off" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "the row is untouched" + ); + } + + /// F1 scenario 6 (#173, the collision case): two warm repos hold identical bytes, + /// which is the shape (forks, a shared LICENSE blob, the empty tree) that makes an + /// exclusive first-pinner claim wrong. Discovery records ONE additive source and + /// sets the incomplete marker, and the marker is what keeps `needs_scan` true so a + /// caller who can only read the OTHER holder is still served. Under an exclusive + /// claim `needs_scan` would be false and that caller would get a 404 for a public + /// object. + #[sqlx::test] + async fn sweep_discovery_multi_holder_serves_both_readers(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["holda", "holdb"]); + let bare_a = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("holda.git"); + + // A is older, so the oldest-first probe order selects it; it is PRIVATE, so it + // denies the anonymous caller. B is public and holds the same bytes. + let mut repo_a = seed_repo(&owner_did, "holda"); + repo_a.is_public = false; + repo_a.created_at = Utc::now() - chrono::Duration::days(2); + let repo_b = seed_repo(&owner_did, "holdb"); + state.db.create_repo(&repo_a).await.expect("seed repo a"); + state.db.create_repo(&repo_b).await.expect("seed repo b"); + + let (raw_cid, _provider) = seed_legacy_pin(&pool, &bare_a, &fx.public_oid, None).await; + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates"); + assert_eq!( + stats.repaired, 1, + "the row is repaired from the first holder" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![repo_a.id.clone()], + "exactly one additive source is recorded, the oldest-first selection" + ); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "the marker records that discovery does not know the full source set" + ); + assert_eq!( + pinned_repo_id(&pool, &fx.public_oid).await, + None, + "no exclusive claim is written for either holder" + ); + + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "a caller who can read only the NON-selected holder is still served" + ); + assert!(body.contains("public bytes"), "the object's bytes serve"); + } + + /// F1 scenario 7 (#173, degenerate state): the cost gate at the top of the row loop + /// fires before the sources query, so a source-less row that is ALREADY raw-CIDv1 + /// never enters discovery and reads nothing. + #[sqlx::test] + async fn sweep_discovery_never_runs_for_a_raw_cidv1_sourceless_row(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["rawnosrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("rawnosrc.git"); + let repo = seed_repo(&owner_did, "rawnosrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + // No provenance recorded: `pin_cid_for` stores the raw key with a NULL repo_id. + let raw_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates"); + + assert_eq!(stats.scanned, 1, "the row is walked"); + assert_eq!(stats.repaired, 0, "a raw-CIDv1 row needs no repair"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "the cost gate spares a raw row every byte read, discovery included" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the raw row is left as-is" + ); + } + + /// F1 scenario 8 (#173, the negative direction of the new branch): a row WITH a + /// recorded source resolves through the existing source loop only. An older warm + /// decoy repo holds identical bytes, so if discovery ran it would record the decoy + /// and set the incomplete marker; neither happens. + #[sqlx::test] + async fn sweep_discovery_is_not_used_for_a_provenanced_row(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["provsrc", "decoysrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provsrc.git"); + + let mut decoy = seed_repo(&owner_did, "decoysrc"); + decoy.created_at = Utc::now() - chrono::Duration::days(2); + let repo = seed_repo(&owner_did, "provsrc"); + state.db.create_repo(&decoy).await.expect("seed decoy"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let (raw_cid, _provider) = + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates"); + + assert_eq!(stats.repaired, 1, "the provenanced row repairs as before"); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the key is rewritten from the recorded source" + ); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 1, + "exactly one read, from the recorded source: discovery never probes" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![repo.id.clone()], + "the decoy is never recorded as a source" + ); + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "a provenanced row's source set is not marked incomplete" + ); + } + + /// F1 scenario 9 (#173, the degradation posture, by execution): if the additive + /// `record_pin_source` fails after the key rewrite lands, the row is raw-CIDv1 with + /// an empty source set. Nothing in the sweep revisits it (the cost gate skips a raw + /// row free on every later pass), so the source record is best-effort and the + /// resolver's own fallback is the healing path, not a retry. This is that state, + /// driven end to end: an empty source set makes `needs_scan` true and the bounded + /// legacy scan still serves the object. + #[sqlx::test] + async fn sweep_repaired_row_without_source_record_still_served(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["nosrcrec"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("nosrcrec.git"); + let repo = seed_repo(&owner_did, "nosrcrec"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // The post-repair state a failed source record leaves behind: raw key, no + // provenance row, no pin_repo_sources row. + let raw_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + assert!( + state + .db + .pin_sources_for_oid(&fx.public_oid) + .await + .unwrap() + .is_empty(), + "the row really has no recorded source" + ); + + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "an empty source set routes the resolver to its bounded legacy scan, so a \ + repaired row whose source record failed still serves" + ); + assert!(body.contains("public bytes"), "the object's bytes serve"); + } + + /// F1 scenario 10 (#173, the unsafe-path arm of the candidate load): a `repos` row + /// whose name cannot be turned into a validated disk path is dropped when the + /// candidate list is built, and the drop is both TERMINAL and NON-FATAL. + /// + /// Terminal: nothing a later pass does makes an unsafe name safe, so the rejection + /// must not mark the row retryable and must not consume a probe against + /// `MAX_LEGACY_DISCOVERY_PROBES`. + /// + /// Non-fatal is the half that matters most. `load_discovery_ctx` builds ONE list for + /// the whole pass, so a rejection that propagated instead of warning would fail the + /// load, `discover_legacy_row` would return Retryable for every source-less row in + /// the pass, and a single unsafe `repos` row anywhere on the node would strand every + /// legacy row behind it on every future run. Here the unsafe row sorts first (the + /// candidate order is oldest-first by `(created_at, id)`), so the pass has to survive + /// it before it can reach the warm holder that actually repairs the row. + #[sqlx::test] + async fn sweep_discovery_drops_unsafe_candidate_and_keeps_going(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["safesrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("safesrc.git"); + + // `validate_repo_name` bails on a '..' sequence, so this name can never resolve + // to a disk path. Asserted against the validator itself rather than assumed, so + // the fixture cannot quietly become a safe name and turn the test vacuous. + let bad_name = "../escape"; + assert!( + crate::git::repo_store::validated_repo_disk_path( + std::path::Path::new("/tmp"), + &owner_did, + bad_name, + ) + .is_err(), + "the fixture name is genuinely refused by the validated resolver" + ); + + let mut unsafe_repo = seed_repo(&owner_did, bad_name); + unsafe_repo.created_at = Utc::now() - chrono::Duration::days(2); + let holder = seed_repo(&owner_did, "safesrc"); + state + .db + .create_repo(&unsafe_repo) + .await + .expect("seed the unsafe repos row"); + state + .db + .create_repo(&holder) + .await + .expect("seed the warm holder"); + + // The pre-provenance shape: NULL repo_id and no pin_repo_sources row. + let (raw_cid, _provider) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("an unsafe candidate never wedges the pass"); + + assert_eq!( + stats.repaired, 1, + "the rejection is non-fatal: a later safe candidate in the same list still \ + repairs the row" + ); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 1, + "the unsafe candidate is dropped before any probe, so the only object read \ + is the warm holder's" + ); + assert_eq!( + stats.retryable_skips, 0, + "an unsafe name is not a condition a later pass clears, so the drop is \ + terminal and drives no cursor rewind" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the key is rewritten from the safe candidate's verified bytes" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![holder.id.clone()], + "only the safe candidate is recorded as a source" + ); + } + + /// Strip Rust line comments, block comments and double-quoted string literals, + /// leaving code. Used by the source scan below, which must not fire on the prose + /// that DESCRIBES the forbidden call (`ipfs_pin.rs` names `repo_store.acquire` in + /// two comments) and must still fire on the call itself. + fn code_only(src: &str) -> String { + let mut out = String::with_capacity(src.len()); + let b: Vec = src.chars().collect(); + let mut i = 0; + while i < b.len() { + if b[i] == '/' && i + 1 < b.len() && b[i + 1] == '/' { + while i < b.len() && b[i] != '\n' { + i += 1; + } + } else if b[i] == '/' && i + 1 < b.len() && b[i + 1] == '*' { + i += 2; + while i + 1 < b.len() && !(b[i] == '*' && b[i + 1] == '/') { + i += 1; + } + i = (i + 2).min(b.len()); + } else if b[i] == '"' { + i += 1; + while i < b.len() && b[i] != '"' { + if b[i] == '\\' { + i += 1; + } + i += 1; + } + i += 1; + } else { + out.push(b[i]); + i += 1; + } + } + out + } + + /// The call-shape half of the sweep's no-remote-fetch guarantee, and a STRUCTURAL + /// guard, not a behavioral one. Say what that means before trusting it. + /// + /// The sweep must never pull a cold repo back from remote storage: it is + /// opportunistic background maintenance over every pinned row on the node, so a + /// fetch would turn a repair pass into a bulk restore. A behavioral test that made + /// the candidate cold, made it fetchable from a live remote, and counted zero fetch + /// attempts cannot be built here, and the reason is worth stating rather than + /// working around: `sweep_pass` takes a `repos_dir`, a `git_bin` and a `&Db`, and + /// holds no `RepoStore` and no `TigrisClient`. A download counter armed on a store + /// the TEST builds could never move no matter what the sweep did, so a zero from it + /// would be vacuous by construction rather than evidence. + /// + /// So the guarantee is asserted from two sides instead. The effect side lives in + /// `sweep_discovery_cold_candidates_do_not_rewind`, which proves the bytes were + /// still available and the cold candidate's path was still absent after two full + /// runs. This is the call side: the module's production code contains none of the + /// fetch-capable call shapes. + /// + /// What it does NOT cover: a fetch reached indirectly through a helper this module + /// calls (`git::store`, `db`) whose own source is not scanned, and git's own lazy + /// fetch if a bare repo on disk were ever configured as a partial clone with a + /// promisor remote. Neither shape exists today; neither is detected here. + #[test] + fn sweep_module_never_calls_a_remote_fetch() { + const SRC: &str = include_str!("ipfs_pin.rs"); + // Scan the PRODUCTION half only. The module's own test module legitimately + // calls `pool.acquire()`, which shares a needle with the store's fetch entry + // points and would otherwise force the needle set to be weakened. + let marker = "\n#[cfg(test)]\nmod tests {"; + let cut = SRC + .find(marker) + .expect("ipfs_pin.rs still opens its test module the usual way"); + let production = &SRC[..cut]; + let code = code_only(production); + + // Anti-vacuity, three ways: a scan that read nothing, a stripper that ate the + // code, or a stripper that left the comments in would each let this pass while + // proving nothing. + assert!( + code.len() > 10_000, + "the scan kept only {} chars of production code, so a clean result proves \ + nothing", + code.len() + ); + assert!( + code.contains("validated_repo_disk_path"), + "the stripper removed real code: the sweep's own path resolver is gone from \ + what was scanned" + ); + assert!( + production.contains("repo_store.acquire"), + "the module no longer names the forbidden call in prose, so this scan is no \ + longer exercising the comment-vs-code distinction it exists to make" + ); + assert!( + !code.contains("bulk restore"), + "the stripper left comments in, so every needle below would fire on the \ + prose that describes it rather than on a call" + ); + + // Every way this crate reaches remote storage. `repo_store::` alone is not a + // needle: the sweep legitimately calls `repo_store::validated_repo_disk_path`, + // the non-fetching path resolver. + for shape in [ + ".acquire(", + "acquire_fresh", + "acquire_write", + "RepoStore", + "TigrisClient", + ".download(", + "tigris", + ] { + assert!( + !code.contains(shape), + "the sweep's production code reaches remote storage through `{shape}`. \ + A repair pass over every pinned row on the node must never pull a cold \ + repo back: that is a bulk restore, not maintenance" + ); + } + } + + /// Poll until some backend in THIS test's database is blocked on a lock, so a test + /// that means to drive an interleaving cannot silently degrade into two calls that + /// simply ran one after the other. Returns false if nothing ever blocked. + async fn wait_for_lock_wait(pool: &PgPool) -> bool { + for _ in 0..600 { + let waiting: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity + WHERE datname = current_database() AND wait_event_type = 'Lock'", + ) + .fetch_one(pool) + .await + .unwrap_or(0); + if waiting > 0 { + return true; + } + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + false + } + + /// GAP 2: discovery's `record_pin_source` then `mark_pin_sources_incomplete` pair + /// under a REAL concurrent writer on the same row, not a reasoned ordering. + /// + /// The order is load-bearing because a `record_pin_source` that actually inserts + /// CLEARS the marker in its own transaction (`rows_affected > 0`), so marking first + /// would have discovery wipe its own marker. The resolver's `needs_scan` is + /// `sources.is_empty() || at_cap || incomplete`, so a non-empty, below-cap, unmarked + /// set is what tells it to stop scanning. Discovery's knowledge is never complete + /// (it stops at the first hit, and its candidate list is capped), so that + /// combination is exactly the state the row must not end in. + /// + /// The interleaving driven here is the one that threatens the pair: a second writer + /// recording a DIFFERENT source for the same oid lands while discovery is mid-row. + /// A row lock parks the sweep inside `repair_legacy_provider_cid`, which is after + /// the source set was read as empty and before either of discovery's own writes, and + /// `wait_for_lock_wait` proves the sweep really is parked rather than already done. + /// The end state must still be a marked row. + #[sqlx::test] + async fn sweep_discovery_marker_survives_a_concurrent_source_record(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["concwarm"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("concwarm.git"); + let holder = seed_repo(&owner_did, "concwarm"); + state.db.create_repo(&holder).await.expect("seed holder"); + // The concurrent writer's repo has no directory on disk, so it is filtered out + // of the candidate list and the only thing it contributes to the row is its own + // `record_pin_source`. + let other = seed_repo(&owner_did, "conccold"); + state.db.create_repo(&other).await.expect("seed other"); + + let (raw_cid, _provider) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + let mut blocker = pool.begin().await.expect("open the blocking transaction"); + sqlx::query("SELECT sha256_hex FROM pinned_cids WHERE sha256_hex = $1 FOR UPDATE") + .bind(&fx.public_oid) + .execute(&mut *blocker) + .await + .expect("hold the row lock"); + + let driver = async { + let parked = wait_for_lock_wait(&pool).await; + state + .db + .record_pin_source(&fx.public_oid, &other.id) + .await + .expect("the concurrent record lands"); + blocker.commit().await.expect("release the row lock"); + parked + }; + let (stats, parked) = tokio::time::timeout(std::time::Duration::from_secs(60), async { + tokio::join!( + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + driver + ) + }) + .await + .expect("the interleaved run terminates"); + + assert!( + parked, + "nothing ever blocked, so the two writers did not actually interleave and \ + this test proved nothing about ordering" + ); + assert_eq!(stats.repaired, 1, "discovery still repairs the row"); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the key is rewritten to the raw-content CID" + ); + let mut sources = state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(); + sources.sort(); + let mut expected = vec![holder.id.clone(), other.id.clone()]; + expected.sort(); + assert_eq!( + sources, expected, + "both writers' sources are present: the record is additive, so neither \ + writer erases the other" + ); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "the marker survives a concurrent record landing inside discovery's window: \ + a non-empty, below-cap, unmarked set would tell the resolver to stop \ + scanning while discovery's knowledge of the set is still incomplete" + ); + assert_eq!( + pinned_repo_id(&pool, &fx.public_oid).await, + None, + "no exclusive first-pinner claim is made under concurrency either" + ); + } + + /// GAP 2, the other interleaving: two sweep passes over the same source-less row at + /// the same time. Whichever order the two passes' `repair_legacy_provider_cid`, + /// `record_pin_source` and `mark_pin_sources_incomplete` calls land in, the end + /// state the resolver reads must be the same one a single pass leaves: the raw key, + /// exactly one recorded source, no exclusive claim, and a marked row. + /// + /// The second pass cannot double-record: `record_pin_source` is + /// `ON CONFLICT DO NOTHING` on `(oid, repo)`, so its insert affects no rows, its + /// marker clear is gated on `rows_affected > 0` and does not run, and its own + /// `mark_pin_sources_incomplete` is idempotent. + #[sqlx::test] + async fn sweep_discovery_two_concurrent_passes_leave_one_marked_source(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["twopass"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("twopass.git"); + let holder = seed_repo(&owner_did, "twopass"); + state.db.create_repo(&holder).await.expect("seed holder"); + let (raw_cid, _provider) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + let (a, b) = tokio::time::timeout(std::time::Duration::from_secs(60), async { + tokio::join!( + crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + &state.db, + ), + crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + &state.db, + ) + ) + }) + .await + .expect("both passes terminate"); + let a = a.expect("the first pass succeeds"); + let b = b.expect("the second pass succeeds"); + assert!( + a.repaired + b.repaired >= 1, + "at least one of the two concurrent passes repairs the row" + ); + + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the key is the raw-content CID whichever pass got there first" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![holder.id.clone()], + "the holder is recorded exactly once: the second pass's insert conflicts and \ + affects no rows" + ); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "two concurrent passes still leave the row marked, so the resolver keeps its \ + fallback scan" + ); + assert_eq!( + pinned_repo_id(&pool, &fx.public_oid).await, + None, + "neither pass makes an exclusive first-pinner claim" + ); + } + + /// GAP 2, the residual this pair does NOT close, pinned here so it is a known + /// property rather than an assumption. + /// + /// `pin_sources_incomplete` is one boolean per OBJECT, not per `(object, repo)`, so + /// any later `record_pin_source` that actually inserts clears it, including one from + /// a repo that has nothing to do with discovery. A concurrent writer whose record + /// commits strictly AFTER discovery's mark therefore leaves the row with a non-empty + /// source set and no marker, which is the combination that stops the resolver's + /// fallback scan. Discovery's ordering cannot prevent that: the clear happens inside + /// the other writer's transaction, after discovery has already finished the row. + /// + /// It is bounded, and the bound is why this is documented rather than fixed here. + /// The clearing writer is a real pusher of the same object, so every source in the + /// set genuinely holds the bytes and the row stays servable through them. And for + /// these rows specifically the state cannot be a regression: before discovery ran, + /// the row was keyed on a provider CID, so the resolver withheld it and + /// `list_pinned_cids` did not advertise it. Closing the residual properly needs a + /// per-`(oid, repo)` marker, which is a change to the marker's shape and not to this + /// sweep. See `Db::record_pin_source`, which states the same residual for the four + /// pre-existing call sites. + #[sqlx::test] + async fn sweep_discovery_marker_is_cleared_by_a_later_record_from_another_repo(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["latewarm"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("latewarm.git"); + let holder = seed_repo(&owner_did, "latewarm"); + state.db.create_repo(&holder).await.expect("seed holder"); + let other = seed_repo(&owner_did, "latecold"); + state.db.create_repo(&other).await.expect("seed other"); + seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates"); + assert_eq!(stats.repaired, 1, "discovery repairs the row"); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "discovery leaves the row marked" + ); + + // A genuine later pusher of the same object from a different repo. + state + .db + .record_pin_source(&fx.public_oid, &other.id) + .await + .expect("the later record lands"); + + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "the residual: a per-object marker is cleared by any later inserting record, \ + so discovery's marker does not survive one" + ); + let mut sources = state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(); + sources.sort(); + let mut expected = vec![holder.id.clone(), other.id.clone()]; + expected.sort(); + assert_eq!( + sources, expected, + "the bound on that residual: every source left in the set is a repo that \ + really holds the object, so the row stays servable through them" + ); + } + /// U4 scenario 6 (#173): `list_pinned_cids` never advertises a key the `/ipfs` /// resolver would withhold. The resolver recomputes the raw CIDv1 from the object /// bytes and 404s any row keyed on a legacy PROVIDER CID, so advertising that key From 25cab140243b7bbe5bac4a4b1e8be4dc22c05de6 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 11 Aug 2026 00:11:13 -0500 Subject: [PATCH 43/77] test(node): pin the per-object marker's third-repo residual record_pin_source's doc already names this residual and accepts it: the pin_sources_incomplete marker is one boolean per object rather than per (object, repo), so a genuine record from a third repo clears a marker that a different repo's failed record set. Its two neighbouring cases are tested, the same repo re-recording and a no-op insert that must not clear, and this one sat between them described in prose and guarded by nothing. The test drives the real path rather than writing rows. Repo A first-pins the object. Repo B genuinely holds it and pushes, its skip-branch record fails on the definite-error arm, and the code marks the set incomplete. An anonymous read is then SERVED from B, which is the assertion that matters: it proves the unrecorded holder is reachable while the marker stands. Repo C pushes, its record succeeds, and the clear fires inside C's transaction. The identical read now 404s, with the preload counter proving the fallback scan never ran. Nothing here changes behavior. It converts an accepted tradeoff from a comment into a guard, so widening it takes a deliberate edit rather than going unnoticed. The doc comment says the residual fails closed, and that holds: the provenance loop and the fallback scan both serve through the same gate_and_serve, so clearing the marker only ever removes a gated search and can never serve an object the caller could not already read. The window is exactly one to fifteen sources, because at cap the count-guarded insert is a no-op and the marker survives. If someone implements the per-(oid, repo) marker the doc proposes, this test is expected to go red. It should be updated to the new behavior, not deleted. Proven load-bearing: removing the clear from record_pin_source leaves the second read at 200 and turns the test red on the marker assertion. --- crates/gitlawb-node/src/test_support.rs | 128 ++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index c3f03f20..073d6cc1 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -4235,6 +4235,134 @@ mod tests { ); } + /// U3 residual (#173): pins the ACCEPTED cost of a single boolean, and is NOT a bug + /// report. `pinned_cids.pin_sources_incomplete` is one flag per OBJECT, not per + /// (object, repo), so a GENUINE record from a third repo C clears a marker that repo + /// B's FAILED record set, and the resolver then reads a non-empty below-cap source + /// set as fully enumerated and stops running the scan fallback. The decision of + /// record is the `record_pin_source` doc comment in `db/mod.rs`, which names this + /// exact case and calls it the deliberate cost of a single boolean; closing it needs + /// a per-(oid, repo) marker table. + /// + /// Why it is acceptable: the residual fails CLOSED. The provenance loop and the scan + /// fallback both serve through the same `gate_and_serve`, so clearing the marker only + /// ever removes a gated search and can never serve something a caller may not read. + /// The window is exactly `1 <= sources < MAX_PIN_SOURCES`: an empty set always scans, + /// and at cap the insert is a no-op so the marker survives. + /// + /// The BEFORE request is what makes the AFTER assertion mean anything: it proves the + /// unrecorded public holder IS reachable while the marker stands, so the later 404 is + /// caused by the clear and by nothing else in the fixture. + /// + /// If someone implements the per-(oid, repo) marker, this test is EXPECTED to go red. + /// Update it to assert the new behavior rather than deleting it: a red here is the + /// residual being closed, not a regression. + #[sqlx::test] + async fn ipfs_cid_third_repo_record_clears_marker_and_hides_an_unrecorded_holder(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3ta", "u3tb", "u3tc"]); + let a_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3ta.git"); + let b_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3tb.git"); + let c_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3tc.git"); + + // Repo A (private) is the first pinner and the only recorded source. + let mut a_repo = seed_repo(&owner_did, "u3ta"); + a_repo.is_public = false; + state.db.create_repo(&a_repo).await.expect("seed A private"); + let cid = pin_cid_for_repo(&a_bare, &fx.public_oid, &state.db, &a_repo.id).await; + state + .db + .record_pin_source(&fx.public_oid, &a_repo.id) + .await + .expect("record the first pinner as a source"); + + // Repo B (public) genuinely holds the object, but its source record never lands, + // so the node marks the set known-incomplete. + let b_repo = seed_repo(&owner_did, "u3tb"); // public, no rule + state.db.create_repo(&b_repo).await.expect("seed B public"); + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &b_bare, &fx.public_oid, &b_repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "B's exhausted record marked the set incomplete" + ); + + // BEFORE: while the marker stands, the scan fallback finds B and serves. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "with the marker set, the unrecorded public holder is reachable" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + + // Repo C (private) pushes the same object. Its record is a GENUINE insert (C is + // not yet a source), so it clears the marker B set, even though B is still missing. + let mut c_repo = seed_repo(&owner_did, "u3tc"); + c_repo.is_public = false; + state.db.create_repo(&c_repo).await.expect("seed C private"); + repin_via_skip_branch(&state, &c_bare, &fx.public_oid, &c_repo.id).await; + + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "a third repo's genuine record clears the per-object marker" + ); + assert!( + !state.db.pin_sources_at_cap(&fx.public_oid).await.unwrap(), + "the set is below cap, so at_cap is not what drives the gate here" + ); + let sources = state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(); + assert_eq!(sources.len(), 2, "the set names A and C only: {sources:?}"); + assert!( + !sources.contains(&b_repo.id), + "B is still missing from the set it was marked for: {sources:?}" + ); + + // AFTER: the identical request now reads the set as complete and 404s the public + // copy it served a moment ago, without ever arming the scan. + crate::api::ipfs::reset_preload_queries(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "the cleared marker hides B: the same anonymous request now 404s" + ); + assert!( + !body.contains("public bytes"), + "the 404 body must not carry the object it declined to serve" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 0, + "the fallback scan provably did not run, so the 404 is the cleared marker and not a failed search" + ); + } + /// U3 scenario 4 (#173): the marker tracks the record's OUTCOME, not the attempt. /// An exhausted retry sets it; a first-attempt success never does. Without the /// second arm the first could be satisfied by marking unconditionally. From bd0bef1476f90e99d9a697e2520c32b954f3ba6f Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:34:13 -0500 Subject: [PATCH 44/77] fix(node): carry discovery onto the per-(oid, repo) incompleteness marker Rebasing onto the #173 head that made the marker per (object, repo) broke three assumptions this branch was written against. Discovery re-marked the set incomplete after recording a discovered source, because a successful record_pin_source used to clear the whole per-object boolean. It no longer does, so that call went from compensating for a clear to being the only thing arming the resolver's fallback. It now marks against the unknown-repo sentinel rather than the repo it just recorded, which would claim that repo failed when it is the one that succeeded. The sentinel is what the v24 migration carries pre-upgrade markers under and means the same thing here: the bounded warm-only probe may have missed a holder and nobody knows which. Two tests asserted the superseded behavior and are inverted rather than deleted, which is what their own comments asked for. GAP 2 documented the per-object marker being cleared by any later record; that residual is closed, so the test now proves discovery's marker survives one. The cold-candidate test proved the cursor was not rewound; the rewind is unconditional on completion now, so it asserts the cost that still matters, that re-walking the row reads no object bytes and restores nothing. --- crates/gitlawb-node/src/ipfs_pin.rs | 18 +++- crates/gitlawb-node/src/test_support.rs | 130 ++++++++++++------------ 2 files changed, 80 insertions(+), 68 deletions(-) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 357502c2..12468b72 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -464,9 +464,21 @@ async fn discover_legacy_row( if let Err(e) = db.record_pin_source(sha, &repo.id).await { tracing::warn!(sha = %sha, repo_id = %repo.id, err = %e, "sweep discovery: failed to record the discovered pin source"); } - // After the record, never before: a successful `record_pin_source` - // clears this marker. - if let Err(e) = db.mark_pin_sources_incomplete(sha).await { + // Discovery found ONE holder out of a bounded, warm-only candidate set, + // so the source set is still not known complete and the resolver must + // keep its scan fallback for this row. + // + // Marked against the UNKNOWN-repo sentinel rather than the repo just + // recorded, which would be a lie (that repo IS recorded). The sentinel is + // the same one the v24 migration carries pre-upgrade markers under, and it + // means what it means here: a source may be missing and nobody knows + // which, so no real record clears it. + // + // Rebase note (#321 onto the per-(oid, repo) marker): the original wrote + // this marker because `record_pin_source` used to clear the whole + // per-object boolean. It no longer does, so this call went from + // compensating for a clear to being the only thing arming the fallback. + if let Err(e) = db.mark_pin_sources_incomplete(sha, "").await { tracing::warn!(sha = %sha, err = %e, "sweep discovery: failed to mark the pin-source set incomplete"); } return DiscoveryOutcome::Repaired; diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 073d6cc1..30cf89e0 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -4235,30 +4235,26 @@ mod tests { ); } - /// U3 residual (#173): pins the ACCEPTED cost of a single boolean, and is NOT a bug - /// report. `pinned_cids.pin_sources_incomplete` is one flag per OBJECT, not per - /// (object, repo), so a GENUINE record from a third repo C clears a marker that repo - /// B's FAILED record set, and the resolver then reads a non-empty below-cap source - /// set as fully enumerated and stops running the scan fallback. The decision of - /// record is the `record_pin_source` doc comment in `db/mod.rs`, which names this - /// exact case and calls it the deliberate cost of a single boolean; closing it needs - /// a per-(oid, repo) marker table. + /// U3 residual, CLOSED (#173 round 12): the incompleteness marker is per + /// `(object, repo)`, so a GENUINE record from a third repo C no longer clears the + /// marker repo B's FAILED record set, and the resolver keeps the scan fallback that + /// finds B's unrecorded public copy. /// - /// Why it is acceptable: the residual fails CLOSED. The provenance loop and the scan - /// fallback both serve through the same `gate_and_serve`, so clearing the marker only - /// ever removes a gated search and can never serve something a caller may not read. - /// The window is exactly `1 <= sources < MAX_PIN_SOURCES`: an empty set always scans, - /// and at cap the insert is a no-op so the marker survives. + /// This test asserted the opposite until the marker moved to `pin_source_failures`. + /// It was written as a deliberate pin on an accepted cost of the single boolean, with + /// a note saying that implementing the per-(oid, repo) marker should turn it red and + /// that it should then be updated rather than deleted. That is what happened, so the + /// assertions are inverted here and the fixture is unchanged. /// /// The BEFORE request is what makes the AFTER assertion mean anything: it proves the - /// unrecorded public holder IS reachable while the marker stands, so the later 404 is - /// caused by the clear and by nothing else in the fixture. - /// - /// If someone implements the per-(oid, repo) marker, this test is EXPECTED to go red. - /// Update it to assert the new behavior rather than deleting it: a red here is the - /// residual being closed, not a regression. + /// unrecorded public holder IS reachable while the marker stands, so an AFTER 404 + /// would be caused by the clear and by nothing else in the fixture. The window this + /// covers is exactly `1 <= sources < MAX_PIN_SOURCES`: an empty set always scans, and + /// at cap the insert is a no-op so the marker survives regardless. #[sqlx::test] - async fn ipfs_cid_third_repo_record_clears_marker_and_hides_an_unrecorded_holder(pool: PgPool) { + async fn ipfs_cid_third_repo_record_keeps_the_marker_and_still_serves_an_unrecorded_holder( + pool: PgPool, + ) { use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); let owner_did = owner.did().to_string(); @@ -4318,19 +4314,20 @@ mod tests { ); // Repo C (private) pushes the same object. Its record is a GENUINE insert (C is - // not yet a source), so it clears the marker B set, even though B is still missing. + // not yet a source), and it must NOT clear the marker B set, because B is still + // missing and C's record says nothing about B. let mut c_repo = seed_repo(&owner_did, "u3tc"); c_repo.is_public = false; state.db.create_repo(&c_repo).await.expect("seed C private"); repin_via_skip_branch(&state, &c_bare, &fx.public_oid, &c_repo.id).await; assert!( - !state + state .db .pin_sources_incomplete(&fx.public_oid) .await .unwrap(), - "a third repo's genuine record clears the per-object marker" + "a third repo's record clears only its own pair, so B's marker survives" ); assert!( !state.db.pin_sources_at_cap(&fx.public_oid).await.unwrap(), @@ -4343,23 +4340,23 @@ mod tests { "B is still missing from the set it was marked for: {sources:?}" ); - // AFTER: the identical request now reads the set as complete and 404s the public - // copy it served a moment ago, without ever arming the scan. + // AFTER: the identical request still serves B's copy, because the surviving + // marker keeps the fallback armed. The scan is asserted to have RUN, so the 200 + // is the fallback finding B and not the provenance loop reaching it some other way. crate::api::ipfs::reset_preload_queries(); let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; assert_eq!( st, - StatusCode::NOT_FOUND, - "the cleared marker hides B: the same anonymous request now 404s" + StatusCode::OK, + "B's marker survived C's record, so the unrecorded public holder is still reachable" ); assert!( - !body.contains("public bytes"), - "the 404 body must not carry the object it declined to serve" + body.contains("public bytes"), + "the served body is the public object's bytes" ); - assert_eq!( - crate::api::ipfs::preload_queries(), - 0, - "the fallback scan provably did not run, so the 404 is the cleared marker and not a failed search" + assert!( + crate::api::ipfs::preload_queries() > 0, + "the fallback scan ran, so the 200 came from the armed fallback" ); } @@ -7772,12 +7769,16 @@ mod tests { /// F1 scenario 4 (#173, MUST-NOT): a candidate that is not on local disk is COLD. /// Discovery must not pull it back from remote storage (the sweep is opportunistic /// background maintenance, not a bulk restore), and it must not mark the row - /// retryable either. The candidate list is every repo on the node rather than the - /// row's holders, so on a Tigris-backed node where most repos are cold a - /// cold-candidate retryable would rewind the cursor - /// (`sweep_legacy_provider_cids` rewinds whenever `retryable_skips > 0`) on every - /// run: the sweep would never drain while paying full discovery cost each pass. The - /// second run's `scanned` is what proves the cursor was not rewound. + /// retryable either. + /// + /// The retryable half was originally about the cursor: a cold-candidate retryable + /// would rewind it, and the second run's `scanned` proved it had not. Round 12 made + /// the rewind unconditional on reaching the end of the table, so every completed run + /// rewinds and the second run re-reads the row whatever this one does. What still + /// holds, and what is asserted below, is the COST: a cold candidate is filtered at + /// load, so re-walking the row reads no object bytes and restores nothing. The + /// retryable-skip assertion also still stands on its own terms, since a cold + /// candidate is not evidence about the row. /// /// The no-fetch half is asserted here on the EFFECT rather than on the call: the /// cold candidate's disk path must still not exist after two full runs, which is @@ -7871,9 +7872,15 @@ mod tests { .await .expect("the second run terminates"); assert_eq!( - (second.scanned, second.retryable_skips), - (0, 0), - "the cursor was not rewound: the second run re-reads nothing" + second.retryable_skips, 0, + "the re-walk still finds nothing retryable about the row" + ); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "the re-walk costs no object read either: the cold candidate is filtered at \ + load on every run, so an unconditional rewind does not turn into repeated \ + discovery reads for this row" ); assert_eq!( stored_pin(&pool, &fx.public_oid).await.0, @@ -8641,28 +8648,21 @@ mod tests { ); } - /// GAP 2, the residual this pair does NOT close, pinned here so it is a known - /// property rather than an assumption. + /// GAP 2, CLOSED (#173 round 12), kept as the regression that proves it stays closed. /// - /// `pin_sources_incomplete` is one boolean per OBJECT, not per `(object, repo)`, so - /// any later `record_pin_source` that actually inserts clears it, including one from - /// a repo that has nothing to do with discovery. A concurrent writer whose record - /// commits strictly AFTER discovery's mark therefore leaves the row with a non-empty - /// source set and no marker, which is the combination that stops the resolver's - /// fallback scan. Discovery's ordering cannot prevent that: the clear happens inside - /// the other writer's transaction, after discovery has already finished the row. + /// This documented a residual: `pin_sources_incomplete` was one boolean per OBJECT, so + /// any later inserting `record_pin_source` cleared it, including one from a repo with + /// nothing to do with discovery, leaving a non-empty source set and no marker, which + /// is the combination that stops the resolver's fallback scan. The marker is now per + /// `(object, repo)` and a record clears only its own pair, so a later writer cannot + /// clear what discovery set. /// - /// It is bounded, and the bound is why this is documented rather than fixed here. - /// The clearing writer is a real pusher of the same object, so every source in the - /// set genuinely holds the bytes and the row stays servable through them. And for - /// these rows specifically the state cannot be a regression: before discovery ran, - /// the row was keyed on a provider CID, so the resolver withheld it and - /// `list_pinned_cids` did not advertise it. Closing the residual properly needs a - /// per-`(oid, repo)` marker, which is a change to the marker's shape and not to this - /// sweep. See `Db::record_pin_source`, which states the same residual for the four - /// pre-existing call sites. - #[sqlx::test] - async fn sweep_discovery_marker_is_cleared_by_a_later_record_from_another_repo(pool: PgPool) { + /// Discovery marks against the unknown-repo sentinel rather than a real repo id, + /// because what it knows is that its bounded warm-only probe may have missed a holder, + /// not that any particular repo failed to record. No real record equals that sentinel, + /// which is what makes the marker survive here. + #[sqlx::test] + async fn sweep_discovery_marker_survives_a_later_record_from_another_repo(pool: PgPool) { use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); let owner_did = owner.did().to_string(); @@ -8711,13 +8711,13 @@ mod tests { .expect("the later record lands"); assert!( - !state + state .db .pin_sources_incomplete(&fx.public_oid) .await .unwrap(), - "the residual: a per-object marker is cleared by any later inserting record, \ - so discovery's marker does not survive one" + "a later record from another repo clears only its own pair, so discovery's \ + marker survives and the resolver keeps its fallback" ); let mut sources = state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(); sources.sort(); From bc202368e13202504fe77534273b508f7559a84a Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:39:11 -0500 Subject: [PATCH 45/77] fix(node): charge discovery's probes against the per-run read budget The two halves of round 12 left a hole between them. The cursor rewind became unconditional on reaching the end of the table, so every completed run re-walks every row, and MAX_DEAD_ROW_READS_PER_RUN is what stops that from costing O(rows) object reads on every boot. But row_read_attempted was only ever set in the provenance loop, so a source-less row, the one shape discovery exists for, spent up to MAX_LEGACY_DISCOVERY_PROBES reads and contributed nothing. The per-row cap bounded one row; nothing bounded the run. discover_legacy_row now reports the probes it spent and the caller charges them whenever the row was not repaired. Per probe rather than per row, because at 16 probes a row the budget would otherwise admit 16 times the reads it names. Including the retryable arm, which is where discovery differs from the provenance loop. There a retryable read is against a repo the row names as a holder and is expected to succeed once that repo warms. Discovery probes repos the row does not name and re-derives its candidate list from scratch each run, so an unrepaired row costs the same reads again on the next boot whatever its outcome was. The cap-reached arm is retryable by design, to stop a grindable repo id burying the true holder, and leaving it uncharged would leave exactly the steerable path uncharged. --- crates/gitlawb-node/src/ipfs_pin.rs | 35 +++-- crates/gitlawb-node/src/test_support.rs | 164 ++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 9 deletions(-) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 12468b72..3af3dc63 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -438,7 +438,8 @@ async fn discover_legacy_row( git_bin: &str, git_timeout: Duration, db: &crate::db::Db, -) -> DiscoveryOutcome { +) -> (DiscoveryOutcome, usize) { + let mut reads = 0usize; if ctx.is_none() { *ctx = Some(match load_discovery_ctx(repos_dir, git_timeout, db).await { Ok(c) => Some(c), @@ -451,7 +452,7 @@ async fn discover_legacy_row( let ctx = match ctx.as_ref().expect("the candidate list was just loaded") { Some(c) => c, // A failed load says nothing about the row, so a later run retries it. - None => return DiscoveryOutcome::Retryable, + None => return (DiscoveryOutcome::Retryable, reads), }; let mut retryable = false; @@ -459,6 +460,9 @@ async fn discover_legacy_row( // MAX_LEGACY_DISCOVERY_PROBES bounds the expensive work exactly. Candidates the // filters already rejected never reach here and so cost nothing against the cap. for (repo, repo_path) in ctx.candidates.iter().take(MAX_LEGACY_DISCOVERY_PROBES) { + // Counted before the match, because the read is spent whatever it returns. This + // is the quantity the caller charges against the per-run budget. + reads += 1; match repair_legacy_provider_cid(repo_path, git_bin, ctx.deadline, sha, db).await { Ok(RepairOutcome::Repaired) => { if let Err(e) = db.record_pin_source(sha, &repo.id).await { @@ -481,7 +485,7 @@ async fn discover_legacy_row( if let Err(e) = db.mark_pin_sources_incomplete(sha, "").await { tracing::warn!(sha = %sha, err = %e, "sweep discovery: failed to mark the pin-source set incomplete"); } - return DiscoveryOutcome::Repaired; + return (DiscoveryOutcome::Repaired, reads); } // The bytes could not be read from this WARM candidate right now, which IS // evidence about the row: try the next one and walk the row again later. @@ -501,12 +505,12 @@ async fn discover_legacy_row( // from grindable owner DIDs, so a terminal verdict would let an attacker bury // the true holder past the cap permanently. The oldest-first order makes that // expensive, and this arm makes it non-permanent. - return DiscoveryOutcome::Retryable; + return (DiscoveryOutcome::Retryable, reads); } if retryable { - DiscoveryOutcome::Retryable + (DiscoveryOutcome::Retryable, reads) } else { - DiscoveryOutcome::Settled + (DiscoveryOutcome::Settled, reads) } } @@ -572,9 +576,10 @@ async fn sweep_pass( // what makes an unrepairable row COST something rather than just being skipped. let mut row_read_attempted = false; if sources.is_empty() { - match discover_legacy_row(&sha, &mut discovery, repos_dir, git_bin, git_timeout, db) - .await - { + let (outcome, reads) = + discover_legacy_row(&sha, &mut discovery, repos_dir, git_bin, git_timeout, db) + .await; + match outcome { DiscoveryOutcome::Repaired => { repaired += 1; row_repaired = true; @@ -582,6 +587,18 @@ async fn sweep_pass( DiscoveryOutcome::Retryable => row_retryable = true, DiscoveryOutcome::Settled => {} } + // Charge every probe that did not end in a repair, INCLUDING a retryable + // one, which is where discovery differs from the provenance loop below. + // There a retryable read is against a repo the row names as a holder, so it + // is expected to succeed once that repo warms. Discovery probes repos the + // row does not name, re-derives its candidate list from scratch on every + // run, and re-probes from the top, so a row that stays unrepaired costs the + // same reads again on the next boot whatever its outcome was. Leaving the + // retryable arm uncharged would also leave the cost open to steering, since + // the cap-reached arm is retryable by design and repo ids are grindable. + if !row_repaired { + dead_row_reads += reads; + } } for repo_id in sources { let repo = match db.get_repo_by_id(&repo_id).await { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 30cf89e0..2eb9746f 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -7893,6 +7893,170 @@ mod tests { ); } + /// #173 round 12 (rebase interaction): discovery's probes count against the per-run + /// fruitless-read budget, PER PROBE rather than per row. + /// + /// The two changes met badly. Round 12 made the cursor rewind unconditional on + /// reaching the end of the table, so every completed run re-walks every row; the + /// budget is what stops a node from paying `O(dead rows)` object reads on every boot. + /// But `row_read_attempted` was only ever set in the provenance loop, so a + /// source-less row, the one shape discovery exists for, spent up to + /// `MAX_LEGACY_DISCOVERY_PROBES` reads and contributed nothing to the budget. The + /// per-row cap bounds one row; nothing bounded the run. + /// + /// Counting per row instead of per probe would not do: at 16 probes a row the budget + /// would admit 16 times the reads it names. Several warm candidates here are what + /// distinguishes the two, since the run must stop after far fewer ROWS than the + /// budget's own number. + #[sqlx::test] + async fn sweep_discovery_probes_count_against_the_fruitless_read_budget(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_DEAD_ROW_READS_PER_RUN; + let batch: i64 = 8; + + // Three warm repos, none of which holds the objects below, so every probe reads + // and finds nothing: three fruitless reads per source-less row. + let names = ["u3bdga", "u3bdgb", "u3bdgc"]; + let _fx = seed_cid_repos(&slug, &short, &names); + for n in names { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + let probes_per_row = names.len(); + + // Source-less legacy rows (no repo_id, no pin_repo_sources) for objects that live + // in none of the repos, which is the shape discovery probes and cannot repair. + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + let rows = cap; // more than the budget allows once each row costs three probes + for i in 0..rows { + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind(format!("{:064x}", i)) + .bind(legacy_dagpb_cid(&raw_cid)) + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + } + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(180), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + batch, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the run terminates"); + + assert!( + stats.dead_row_reads >= cap, + "discovery's fruitless probes reach the budget (spent {})", + stats.dead_row_reads + ); + // Per PROBE, not per row: at three probes a row the run must stop after roughly a + // third of the budget's worth of rows, plus at most one batch of overshoot. + // Asserted BEFORE the coarser bound below so that a per-row implementation + // reddens on the property this test is named for. The fixture deliberately holds + // exactly `cap` rows, so per-row counting walks the whole table and would + // otherwise trip the coarse assertion first, reporting the wrong reason. + assert!( + stats.scanned <= cap / probes_per_row + batch as usize, + "the budget counts probes, not rows: scanned {} with a cap of {cap} at \ + {probes_per_row} probes per row", + stats.scanned + ); + assert!( + stats.scanned < rows, + "the run stops short of the table instead of walking all {rows} rows \ + (scanned {})", + stats.scanned + ); + assert_ne!( + state.db.pin_repair_cursor().await.unwrap(), + "", + "a run that stops on its budget keeps its place for the next one" + ); + } + + /// #173 round 12: the RETRYABLE arm of discovery is charged to the budget too, which + /// is the half that differs from the provenance loop and the half an attacker can + /// steer. + /// + /// The cap-reached outcome is retryable BY DESIGN, so that a grindable repo id cannot + /// bury the true holder past the cap permanently. That same design makes it the arm a + /// hostile registrant can hold a row in: register more than the cap's worth of warm + /// repos and every source-less row costs a full cap of reads, on every boot, forever. + /// Charging only the settled arm would leave exactly that uncharged. + #[sqlx::test] + async fn sweep_discovery_charges_a_retryable_cap_reached_row(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let probe_cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + // One more warm repo than the probe cap, so the walk stops with candidates left + // and classifies the row RETRYABLE rather than settled. None holds the object. + let names: Vec = (0..probe_cap + 1).map(|i| format!("u3ret{i}")).collect(); + let name_refs: Vec<&str> = names.iter().map(|s| s.as_str()).collect(); + let _fx = seed_cid_repos(&slug, &short, &name_refs); + for n in &names { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind("a".repeat(64)) + .bind(legacy_dagpb_cid(&raw_cid)) + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the run terminates"); + + assert_eq!( + stats.retryable_skips, 1, + "the cap was reached with candidates left, so the row is retryable" + ); + assert_eq!( + stats.repaired, 0, + "no candidate holds the object, so nothing is repaired" + ); + assert_eq!( + stats.dead_row_reads, probe_cap, + "a retryable cap-reached row is charged its full cap of probes, not zero" + ); + } + /// F1 scenario 5 (#173, BOUND plus anti-burial): the probe cap counts the expensive /// unit, a bounded object read from a warm repo, so a row costs at most /// `MAX_LEGACY_DISCOVERY_PROBES` reads however many candidates the node holds. With From 4c665fbca8641f7dfe6770d1ee5384a0af3d215c Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:46:29 -0500 Subject: [PATCH 46/77] fix(node): port discovery's candidate load onto the paged repo query #173 removed Db::list_all_repos in favour of a keyset-paged list_repos_page_for_scan, so this branch stopped compiling on rebase. One call site, load_discovery_ctx, ported with its reviewed semantics intact. The quarantine drop gets simpler rather than different: each page row carries its own quarantined flag, so the separate list_quarantined_repos query is gone and the filter runs over rows the pass already read. Private non-quarantined repos still stay in the list. Warm-only via validated_repo_disk_path inside spawn_blocking is untouched, so nothing here reaches repo_store.acquire and a repair pass still cannot become a bulk restore, and an unsafe path is still dropped with the same warn and stays terminal. Oldest-first by (created_at, id) now comes from the query's ORDER BY (index backed by migration v25) instead of a client-side sort, and pages concatenate in that order, so the explicit sort_by is redundant and removed. The anti-grinding reason for that order is unchanged and the doc comment keeps it: repo_id derives from a grindable owner DID, so an id sort would let someone bury the true holder past the probe cap. The probe cap still bounds expensive reads rather than candidates considered, and no cap on candidates was added. This pager runs to exhaustion, unlike the resolver's, which drives the same query and stops when its probe or visit budget is spent. The comment says why: the resolver is an anonymously reachable route holding scarce walk admission, while this is background maintenance on a timer with no caller and no permit contention that needs the whole warm candidate set to settle a row. Paging is per pass, matching load_discovery_ctx's existing once-per-pass lifetime. Refs Gitlawb/node#321. --- crates/gitlawb-node/src/ipfs_pin.rs | 63 +++++++++++++++++++---------- 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 3af3dc63..5d5e08fd 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -330,10 +330,11 @@ struct DiscoveryCtx { /// Three filters, all applied before any probe so a rejected candidate costs nothing /// against [`MAX_LEGACY_DISCOVERY_PROBES`]: /// -/// - QUARANTINE. `list_all_repos` is a bare SELECT with no visibility or quarantine -/// filter, and a quarantined repo is hidden from every reader, so it must not become -/// a discovery source either. The resolver's legacy scan loads the same set and gates -/// on it. Private, non-quarantined repos DO stay in the list: an additive source +/// - QUARANTINE. A quarantined repo is hidden from every reader, so it must not become +/// a discovery source either. Each page row carries its own `quarantined` flag, so +/// the drop is a filter over the rows this pass already read rather than a second +/// whole-node query. The resolver's legacy scan reads the same rows and drops on the +/// same flag. Private, non-quarantined repos DO stay in the list: an additive source /// record binds nothing to one repo's ACL, because the resolver gates every source /// independently at serve time, so probing a private repo leaks nothing. /// - WARM ONLY. The path is resolved through the repo store's validated resolver and @@ -344,32 +345,50 @@ struct DiscoveryCtx { /// - UNSAFE PATH. A name that fails the validated resolver is dropped with a warn and /// is terminal; nothing a later run changes. /// -/// The survivors are ordered oldest-first by `(created_at, id)` rather than by id +/// The candidates are ordered oldest-first by `(created_at, id)` rather than by id /// alone. `repo_id` derives from the owner DID, which anyone can grind, so an id sort /// would let an attacker register low-sorting repos and push the true holder past the /// probe cap. Source-less rows predate provenance and their holders are old repos, -/// while freshly registered repos sort last and cannot be backdated. +/// while freshly registered repos sort last and cannot be backdated. That order is now +/// the QUERY's (`ORDER BY created_at ASC, id ASC`, index-backed by migration v25) and +/// pages concatenate in it, so the list is globally sorted as it is built and no +/// client-side sort is involved. async fn load_discovery_ctx( repos_dir: &std::path::Path, git_timeout: Duration, db: &crate::db::Db, ) -> Result { - let repos = db.list_all_repos().await?; - let quarantined: std::collections::HashSet = db - .list_quarantined_repos() - .await? - .into_iter() - .map(|r| r.id) - .collect(); - let mut candidates: Vec = repos - .into_iter() - .filter(|r| !quarantined.contains(&r.id)) - .collect(); - candidates.sort_by(|a, b| { - a.created_at - .cmp(&b.created_at) - .then_with(|| a.id.cmp(&b.id)) - }); + // Page to EXHAUSTION. The resolver's legacy scan drives this same query and + // deliberately does NOT (`api::ipfs`): it stops the moment its probe or visit budget + // is spent, because it runs on an anonymously reachable route while holding scarce + // walk admission, where reading the whole table is the amplification the budget + // exists to forbid. Same query, different threat model. This is background + // maintenance on a timer: no caller to amplify, no permit to pin, and the pass needs + // the whole warm candidate set before it can call a row settled. Do not "align" this + // loop with the resolver's — the budgets it stops on have no counterpart here. + // + // Per PASS, not per row: `load_discovery_ctx` already runs once per pass and its + // result is reused for every source-less row, so the paging cost is paid once. + let page_rows = crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS; + let mut cursor: Option<(String, String)> = None; + let mut candidates: Vec = Vec::new(); + loop { + let page = db + .list_repos_page_for_scan( + cursor + .as_ref() + .map(|(created_at, id)| (created_at.as_str(), id.as_str())), + page_rows as i64, + ) + .await?; + let Some(last) = page.last() else { break }; + cursor = Some((last.created_at_key.clone(), last.repo.id.clone())); + let last_page = page.len() < page_rows; + candidates.extend(page.into_iter().filter(|r| !r.quarantined).map(|r| r.repo)); + if last_page { + break; + } + } let repos_dir = repos_dir.to_path_buf(); let warm = tokio::task::spawn_blocking(move || { From 650460acc049a06de42d3e3747ea7cf5d184c82e Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:56:56 -0500 Subject: [PATCH 47/77] test(node): cover the pre-provenance upgrade row end to end The sweep's source-less discovery had no fixture built from a real pre-migration row. Every existing test seeded its pin through the modern path, which cannot reproduce the shape an upgraded node actually carries: a row written when pinned_cids.repo_id and pin_repo_sources did not exist yet, so its source set is empty with no other signal. The fixture un-applies migrations 19 and 20, inserts the provider-keyed pin naming only (sha256_hex, cid, pinned_at), then migrates forward. That ordering is what makes it real; seed_legacy_pin binds repo_id, the column being proven absent, so the insert is inline. It brackets the repair rather than asserting an end state: the anonymous GET is driven before the sweep and asserted not to serve, then after the sweep asserted to return the object's bytes. The marker assertion queries pin_source_failures directly for the "" sentinel instead of reading the pin_sources_incomplete bool, which cannot distinguish the sentinel from a repo id that record_pin_source would later clear. Verified by mutation: short-circuiting the sources.is_empty() discovery arm restores the pre-fold fall-through and the test goes red on the repair assertion. --- crates/gitlawb-node/src/test_support.rs | 198 ++++++++++++++++++++++++ 1 file changed, 198 insertions(+) diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 2eb9746f..e8c6dd13 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -8378,6 +8378,204 @@ mod tests { assert!(body.contains("public bytes"), "the object's bytes serve"); } + /// U2 (#173, F1 proved by execution): a genuinely PRE-PROVENANCE `pinned_cids` row, + /// one written while `pinned_cids.repo_id` and `pin_repo_sources` do not exist, is + /// repaired by the sweep and then served end to end by `GET /ipfs/{cid}`. + /// + /// The fixture un-applies v19 and v20 before the insert on purpose. A source-less + /// row written through the modern schema is a shortcut: it shows the sweep copes + /// with an empty source set, not that it copes with the row shape an upgraded node + /// actually carries. With the column absent, a provenance-carrying insert is + /// impossible, so the row cannot be anything but the real upgrade case. + /// + /// F1 was that the sweep SKIPPED exactly this row. An empty `pin_sources_for_oid` + /// left the `for repo_id in sources` body unentered while the cursor had already + /// advanced past it, so the row kept its provider key and stayed unresolvable with + /// nothing left to fix it. The pre-sweep assertion below brackets the repair, so + /// the serve at the end cannot pass vacuously on a row that was already fine. + #[sqlx::test] + async fn sweep_repairs_pre_provenance_upgrade_row_and_serves(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["preprov"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("preprov.git"); + let repo = seed_repo(&owner_did, "preprov"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + + // Un-apply the provenance schema: the node is back at the shape it had before + // v19 and v20, where a pin could not carry provenance at all. + sqlx::query("DROP TABLE IF EXISTS pin_repo_sources") + .execute(&pool) + .await + .unwrap(); + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS repo_id") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version IN (19, 20)") + .execute(&pool) + .await + .unwrap(); + + // The legacy row, INSERTed naming only the columns that exist at this schema. + // `seed_legacy_pin` cannot be reused here: it binds `repo_id`, which is the one + // thing this fixture is proving the row never had. + let (_ty, bytes) = crate::git::store::read_object(&bare, &fx.public_oid) + .expect("read object bytes") + .expect("object exists in the bare repo"); + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&bytes).to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + assert_ne!( + provider_cid, raw_cid, + "the legacy key differs from the raw resolver key" + ); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind(&fx.public_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + // Upgrade forward. The row now reads as the exact F1 shape: no first-pinner, no + // source rows, and no incompleteness signal either, so emptiness is the only + // thing the sweep has to go on. + state + .db + .run_migrations() + .await + .expect("re-apply the provenance migrations"); + let first_pinner: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = $1") + .bind(&fx.public_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert!( + first_pinner.is_none(), + "the upgraded row carries no first-pinner: the column did not exist when it \ + was written" + ); + assert!( + state + .db + .pin_sources_for_oid(&fx.public_oid) + .await + .unwrap() + .is_empty(), + "the upgraded row has no recorded source of any kind" + ); + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "and no incompleteness marker either: an upgrade row is indistinguishable \ + from a healthy one except by being empty" + ); + + // The bracket: while the row is unrepaired the resolver withholds it, so the + // raw key a correct client sends does not serve. + let (st_before, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st_before, + StatusCode::OK, + "the raw key does not serve while the row is still keyed on the provider CID" + ); + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ) + .await; + assert_eq!( + stats.repaired, 1, + "the sweep repairs the pre-provenance upgrade row" + ); + + let (stored, stashed) = stored_pin(&pool, &fx.public_oid).await; + assert_eq!( + stored, raw_cid, + "the key is rewritten to the raw-content resolver key" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed rather than dropped" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![repo.id.clone()], + "the repo discovery read the bytes from is recorded as a source" + ); + let claimed: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = $1") + .bind(&fx.public_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert!( + claimed.is_none(), + "discovery records the holder ADDITIVELY: reading identical bytes proves \ + the repo holds the object, never that it pinned it first, so the exclusive \ + first-pinner column stays NULL" + ); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "one discovered holder out of a bounded candidate set never proves the set \ + complete, so the resolver keeps its scan fallback for this row" + ); + let marker: Vec = + sqlx::query_scalar("SELECT repo_id FROM pin_source_failures WHERE sha256_hex = $1") + .bind(&fx.public_oid) + .fetch_all(&pool) + .await + .unwrap(); + assert_eq!( + marker, + vec![String::new()], + "the marker is against the UNKNOWN-repo sentinel, not the repo just \ + recorded: no real record can clear it" + ); + + // End to end: the repaired key serves the object's raw bytes to an anonymous + // caller, which is the whole point of repairing it. + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "the repaired raw key serves"); + assert_eq!( + body, bytes, + "the served body is the object's raw content, byte for byte" + ); + } + /// F1 scenario 10 (#173, the unsafe-path arm of the candidate load): a `repos` row /// whose name cannot be turned into a validated disk path is dropped when the /// candidate list is built, and the drop is both TERMINAL and NON-FATAL. From 22560b0872bd16dd66100681b1050faa677fc568 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:20:19 -0500 Subject: [PATCH 48/77] fix(node): give each discovery row its own slice of the pass budget DiscoveryCtx held one deadline for a whole sweep pass, so a single hung candidate on the first source-less row spent it for every row behind that one. Those rows then called repair_legacy_provider_cid with an expired deadline, came back retryable without a meaningful probe, and were charged reads for work they could not do. CID order is stable, so the same first row consumed the budget on every boot and the rows behind it starved permanently. Each row now probes under min(now + git_timeout / 4, pass_deadline), so at least four rows per pass get a live deadline and no row can take more than a quarter. A row that starts with the pass deadline already spent returns PassBudgetSpent: no probe, zero reads charged, retryable, and surfaced on SweepStats so a run can say it happened. Charging those reads would burn MAX_DEAD_ROW_READS_PER_RUN on probes that cannot succeed. The field is renamed to pass_deadline because that is what it is: load_discovery_ctx runs once per pass and a run loops passes. Naming it for the run would have stated a bound the code does not hold. The divisor is not a bound on a single probe. A row's slice is still shared by up to MAX_LEGACY_DISCOVERY_PROBES candidates, and the doc says so rather than implying a guarantee. Verified by mutation: collapsing the row deadline back to the pass deadline starves the second row, and folding the spent-budget arm into ordinary retryable accounting charges reads that must stay free. --- crates/gitlawb-node/src/ipfs_pin.rs | 87 ++++++++++- crates/gitlawb-node/src/test_support.rs | 199 ++++++++++++++++++++++++ 2 files changed, 278 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 5d5e08fd..c754d051 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -297,6 +297,11 @@ pub(crate) struct SweepStats { /// bytes are gone, so the read is pure waste and the next run will waste it again. /// [`MAX_DEAD_ROW_READS_PER_RUN`] bounds this per run. pub dead_row_reads: usize, + /// Whether at least one source-less row was reached with the pass's whole discovery + /// budget already spent, so it was skipped without a probe (see + /// [`DISCOVERY_ROW_BUDGET_DIVISOR`]). Reporting only, like `retryable_skips`: it + /// drives no control decision, it just keeps a starved pass from being silent. + pub discovery_budget_spent: bool, } /// How many fruitless object reads one sweep run will spend before it stops and leaves @@ -309,8 +314,32 @@ pub(crate) struct SweepStats { /// early keeps the cursor, so the next boot resumes past the rows already walked rather /// than repeating them, and the table still gets covered across boots. pub(crate) const MAX_DEAD_ROW_READS_PER_RUN: usize = 64; + +/// How the pass's discovery budget is sliced per source-less row (#173 round 13, F6). +/// +/// The pass budget alone is not enough. It is one `git_timeout` shared by every +/// source-less row the pass reaches, so a single wedged candidate on the first row spent +/// all of it and every later row arrived with a dead deadline, came back retryable +/// without a real probe, and starved. `sha256_hex` order is stable, so the same row won +/// the race on every boot and the rows behind it were never probed at all. +/// +/// The trade this number sets, stated both ways so neither half is silent: at least four +/// rows are guaranteed a live probe out of one pass budget, and no single row may spend +/// more than a quarter of it. Raising the divisor guarantees more rows per pass and gives +/// each a shorter probe; lowering it does the reverse. Four keeps a row's slice generous +/// against the default 600s `git_service_timeout_secs` (150s, far past any healthy +/// `cat-file`) while still bounding the damage one wedged candidate can do. +/// +/// It is NOT a bound on a single probe. Within a row's slice the per-row deadline is +/// shared by up to [`MAX_LEGACY_DISCOVERY_PROBES`] candidates exactly as the pass +/// deadline used to be shared by rows, so one wedged candidate can still consume its +/// row's whole slice and leave that row's later candidates unprobed. What this bounds is +/// the blast radius: the row, not the pass. +const DISCOVERY_ROW_BUDGET_DIVISOR: u32 = 4; + /// The warm, non-quarantined repos one pass may probe for a source-less legacy row, -/// plus the one absolute deadline every probe in the pass shares. +/// plus the absolute deadline bounding the pass's discovery as a whole, from which each +/// row takes a slice. /// /// Loaded LAZILY, once per pass, on the first source-less row, mirroring the resolver's /// own legacy-scan context: a pass with no such row pays nothing. The `is_dir` warm @@ -320,9 +349,14 @@ struct DiscoveryCtx { /// Warm candidates with their validated disk paths, oldest-first by /// `(created_at, id)`. candidates: Vec<(crate::db::RepoRecord, std::path::PathBuf)>, - /// Shared by every discovery read in the pass, so one pass's discovery costs at - /// most one `git_timeout` in total on top of the per-row probe cap. - deadline: Instant, + /// The ceiling on the whole pass's discovery, so one pass costs at most one + /// `git_timeout` in total on top of the per-row probe cap. Per PASS, not per run: + /// `load_discovery_ctx` runs once per `sweep_pass` and a run loops passes. + /// + /// No row gets all of it. Each takes at most + /// `git_timeout / DISCOVERY_ROW_BUDGET_DIVISOR`, clamped to what is left here, and a + /// row reached with this already past is skipped without a probe. + pass_deadline: Instant, } /// Build one pass's discovery candidate list. @@ -416,7 +450,7 @@ async fn load_discovery_ctx( Ok(DiscoveryCtx { candidates: warm, - deadline: Instant::now() + git_timeout, + pass_deadline: Instant::now() + git_timeout, }) } @@ -430,6 +464,12 @@ enum DiscoveryOutcome { Retryable, /// The row's key was rewritten from bytes verified in a warm local repo. Repaired, + /// The pass's whole discovery budget was already spent when this row was reached, so + /// nothing was probed. Accounted RETRYABLE like the arm above (nothing was learned + /// about the row), but kept distinct because it must cost NOTHING: charging it the + /// reads it never made would burn [`MAX_DEAD_ROW_READS_PER_RUN`] on rows that were + /// only ever skipped, pausing the run early for no information. + PassBudgetSpent, } /// Probe a bounded set of warm local repos for a source-less legacy row's object. @@ -474,6 +514,19 @@ async fn discover_legacy_row( None => return (DiscoveryOutcome::Retryable, reads), }; + // Reached with the pass's discovery budget already gone: probing now buys nothing but + // a spent-deadline error per candidate, so return before any read is charged. + if Instant::now() >= ctx.pass_deadline { + return (DiscoveryOutcome::PassBudgetSpent, reads); + } + // This row's slice of the pass budget, clamped to what is left of it. Without the + // clamp a row reached near the end of the pass would overrun the pass's own ceiling; + // without the slice one wedged candidate would spend the whole pass on this row. + let row_deadline = std::cmp::min( + Instant::now() + git_timeout / DISCOVERY_ROW_BUDGET_DIVISOR, + ctx.pass_deadline, + ); + let mut retryable = false; // Every candidate that gets this far is READ, so taking the first // MAX_LEGACY_DISCOVERY_PROBES bounds the expensive work exactly. Candidates the @@ -482,7 +535,7 @@ async fn discover_legacy_row( // Counted before the match, because the read is spent whatever it returns. This // is the quantity the caller charges against the per-run budget. reads += 1; - match repair_legacy_provider_cid(repo_path, git_bin, ctx.deadline, sha, db).await { + match repair_legacy_provider_cid(repo_path, git_bin, row_deadline, sha, db).await { Ok(RepairOutcome::Repaired) => { if let Err(e) = db.record_pin_source(sha, &repo.id).await { tracing::warn!(sha = %sha, repo_id = %repo.id, err = %e, "sweep discovery: failed to record the discovered pin source"); @@ -557,6 +610,7 @@ async fn sweep_pass( let mut repaired = 0usize; let mut retryable_skips = 0usize; let mut dead_row_reads = 0usize; + let mut discovery_budget_spent = false; let mut last = cursor; // Loaded on the first source-less row and reused by every later one. The outer // `None` is "not loaded yet"; `Some(None)` is "the load failed this pass", which is @@ -605,6 +659,13 @@ async fn sweep_pass( } DiscoveryOutcome::Retryable => row_retryable = true, DiscoveryOutcome::Settled => {} + // Worth walking again, like any retryable row, but it read nothing and so + // is charged nothing below (`reads` is zero). The flag is what keeps a + // starved pass from being silent. + DiscoveryOutcome::PassBudgetSpent => { + row_retryable = true; + discovery_budget_spent = true; + } } // Charge every probe that did not end in a repair, INCLUDING a retryable // one, which is where discovery differs from the provenance loop below. @@ -704,6 +765,7 @@ async fn sweep_pass( passes: 1, retryable_skips, dead_row_reads, + discovery_budget_spent, }) } @@ -732,8 +794,9 @@ pub(crate) async fn sweep_legacy_provider_cids_once( /// /// A row with NO recorded source is the pre-provenance case this exists for, so it is /// not skipped: the pass probes a bounded, quarantine-filtered set of WARM local repos -/// for the object (at most [`MAX_LEGACY_DISCOVERY_PROBES`] reads per row, sharing one -/// per-pass deadline) and, on a hit, rewrites the key from the verified bytes and +/// for the object (at most [`MAX_LEGACY_DISCOVERY_PROBES`] reads per row, each row +/// taking a [`DISCOVERY_ROW_BUDGET_DIVISOR`] slice of the pass's one discovery deadline) +/// and, on a hit, rewrites the key from the verified bytes and /// records the discovered repo ADDITIVELY alongside the incomplete marker. It never /// writes an exclusive first-pinner claim and never pulls a cold repo back from remote /// storage. See `discover_legacy_row` for why both of those matter. @@ -791,6 +854,7 @@ pub(crate) async fn sweep_legacy_provider_cids( totals.repaired += pass.repaired; totals.retryable_skips += pass.retryable_skips; totals.dead_row_reads += pass.dead_row_reads; + totals.discovery_budget_spent |= pass.discovery_budget_spent; totals.passes += 1; // A short batch means the ordered walk reached the end of the table. Stop here // rather than after an extra empty pass, and do NOT sleep on the way out. @@ -811,6 +875,13 @@ pub(crate) async fn sweep_legacy_provider_cids( } tokio::time::sleep(delay).await; } + if totals.discovery_budget_spent { + tracing::info!( + passes = totals.passes, + "legacy provider-CID sweep: a pass spent its whole discovery budget before \ + reaching every source-less row; the rest were skipped unprobed" + ); + } if completed { if let Err(e) = db.set_pin_repair_cursor("").await { tracing::warn!(err = %e, "failed to rewind the legacy provider-CID sweep cursor"); diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index e8c6dd13..0dcef1a0 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -8125,6 +8125,205 @@ mod tests { ); } + /// Write an executable `git` stand-in and return its path. + fn write_git_shim(name: &str, script: &str) -> std::path::PathBuf { + let path = std::env::temp_dir().join(name); + std::fs::write(&path, script).expect("write the git shim"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)) + .expect("chmod the git shim"); + } + path + } + + /// F6 scenario 1 (#173 round 13): one hung candidate must not starve the rows behind + /// it in the same pass. `DiscoveryCtx` is loaded once per pass, so before the per-row + /// slice every source-less row in a pass shared ONE deadline: the first row's wedged + /// `cat-file` spent the whole budget, and every later row reached + /// `repair_legacy_provider_cid` with it already gone, came back retryable without a + /// meaningful probe, and (because `sha256_hex` order is stable) starved on the same + /// row on every boot. + /// + /// Two source-less legacy rows, one warm candidate holding both objects, and a `git` + /// stand-in that wedges on the FIRST row's object and answers the second's for real. + /// With `git_timeout` at 4s the row slice is 1s, so the wedged row costs a quarter of + /// the pass budget and the second row still probes with a live deadline. RED before + /// the slice: the first row burns all 4s and the second is never repaired. + #[sqlx::test] + async fn sweep_discovery_hung_candidate_does_not_starve_later_rows(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["hungsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("hungsrc.git"); + let repo = seed_repo(&owner_did, "hungsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // The walk is ordered by `sha256_hex`, so the row that is reached FIRST is the + // lexicographically smaller oid. That is the one the stand-in wedges on. + let mut oids = [fx.public_oid.clone(), fx.secret_oid.clone()]; + oids.sort(); + let (hung_oid, live_oid) = (oids[0].clone(), oids[1].clone()); + let (_hung_raw, hung_provider) = seed_legacy_pin(&pool, &bare, &hung_oid, None).await; + let (live_raw, live_provider) = seed_legacy_pin(&pool, &bare, &live_oid, None).await; + + // The type stage feeds the oid on STDIN (`cat-file --batch-check`) and the + // content stage puts it in argv, so the stand-in has to look in both places. + let git_bin = write_git_shim( + &format!("gl-hung-git-{short}"), + &format!( + "#!/bin/sh\n\ + if [ \"$2\" = \"--batch-check\" ]; then\n\ + \x20 oid=$(cat)\n\ + \x20 case \"$oid\" in\n\ + \x20 {hung_oid}) sleep 30; exit 1 ;;\n\ + \x20 esac\n\ + \x20 printf '%s\\n' \"$oid\" | git \"$@\"\n\ + \x20 exit $?\n\ + fi\n\ + case \"$*\" in\n\ + \x20 *{hung_oid}*) sleep 30; exit 1 ;;\n\ + esac\n\ + exec git \"$@\"\n" + ), + ); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(60), + crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + git_bin.to_str().unwrap(), + std::time::Duration::from_secs(4), + 16, + &state.db, + ), + ) + .await + .expect("the pass terminates") + .expect("the pass succeeds"); + + assert_eq!(stats.scanned, 2, "both rows are walked in the one pass"); + assert_eq!( + stored_pin(&pool, &live_oid).await.0, + live_raw, + "the second row still probes with a LIVE deadline and is repaired in the \ + same pass; a hung first row must not spend the whole pass budget" + ); + assert_eq!(stats.repaired, 1, "exactly the second row is repaired"); + assert_eq!( + stored_pin(&pool, &hung_oid).await.0, + hung_provider, + "the wedged row keeps its provider key" + ); + assert_eq!( + stats.retryable_skips, 1, + "the wedged row is retryable, so a later run walks it again" + ); + assert_ne!( + live_raw, live_provider, + "control: the repaired key really differs from the seeded legacy one" + ); + } + + /// F6 scenario 2 (#173 round 13, MUST-NOT): once a pass's whole discovery budget is + /// spent, the rows it has not reached are skipped CHEAPLY and visibly, never folded + /// into ordinary retryable accounting. A row charged for a probe it never meaningfully + /// made burns `MAX_DEAD_ROW_READS_PER_RUN` on nothing, which pauses the run early and + /// (once the discovery continuation lands) would let it advance over windows nobody + /// probed. + /// + /// Seven source-less legacy rows, one warm candidate, and a `git` that wedges on + /// everything. With `git_timeout` at 4s each row slice is 1s, so about four rows spend + /// the pass budget between them and the rest start with it already gone: those charge + /// ZERO reads and the pass reports that it ran out. RED before the skip arm: every row + /// past the first reaches the probe with a dead deadline and is charged a read for it, + /// so `dead_row_reads` equals the row count. + #[sqlx::test] + async fn sweep_discovery_spent_pass_budget_skips_cheaply(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["spentsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("spentsrc.git"); + let repo = seed_repo(&owner_did, "spentsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let oids = [ + fx.public_oid.clone(), + fx.secret_oid.clone(), + fx.public_tree_oid.clone(), + fx.secret_tree_oid.clone(), + fx.root_tree_oid.clone(), + fx.commit_oid.clone(), + fx.tag_oid.clone(), + ]; + for oid in &oids { + seed_legacy_pin(&pool, &bare, oid, None).await; + } + + // Wedges on every invocation, so no row can ever be repaired and the only + // question left is what each one COSTS. + let git_bin = write_git_shim( + &format!("gl-spent-git-{short}"), + "#!/bin/sh\nsleep 30\nexit 1\n", + ); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(60), + crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + git_bin.to_str().unwrap(), + std::time::Duration::from_secs(4), + 16, + &state.db, + ), + ) + .await + .expect("the pass terminates") + .expect("the pass succeeds"); + + assert_eq!(stats.scanned, oids.len(), "every row is walked"); + assert_eq!(stats.repaired, 0, "a wedged candidate repairs nothing"); + assert!( + stats.dead_row_reads < stats.scanned, + "a row reached after the pass budget is spent must be skipped free, not \ + charged for a probe it cannot make: {} reads charged over {} rows", + stats.dead_row_reads, + stats.scanned + ); + assert!( + stats.dead_row_reads <= 5, + "the pass budget is four row slices wide, so at most the rows that really \ + probed are charged (plus at most one on the boundary); got {}", + stats.dead_row_reads + ); + assert!( + stats.discovery_budget_spent, + "a pass that ran out of discovery budget must SAY so rather than starving \ + its remaining rows silently" + ); + assert_eq!( + stats.retryable_skips, + oids.len(), + "no row is settled: the wedged ones and the unprobed ones are all worth \ + walking again" + ); + } + /// F1 scenario 6 (#173, the collision case): two warm repos hold identical bytes, /// which is the shape (forks, a shared LICENSE blob, the empty tree) that makes an /// exclusive first-pinner claim wrong. Discovery records ONE additive source and From 4fdfa1cedb3dcb6d7d6cc4004e7277a9fa95de94 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:46:03 -0500 Subject: [PATCH 49/77] fix(node): rotate discovery candidates across traversals Discovery probed the first sixteen warm candidates of a stable (created_at, id) ordering on every pass, so an object held only by the seventeenth repo was never repaired and every boot repeated the same work. The probe cap bounds expensive reads, which is right; it was also silently bounding which candidates are ever considered. A persisted continuation now rotates the window. Four choices carry it, each forced by a way an earlier draft was wrong: Keyset, not offset. An offset indexes into a list whose length anyone can change, so candidates entering below the boundary shift the window off the holder. A key names a position that survives insertions. created_at is server-stamped, so minted repos only ever append. Per traversal, not per pass. The row cursor also advances per pass, so per-pass rotation gives row R the window index (t*m + j) mod W and a row sees only W/gcd(m, W) of the W windows. Sharing one window per traversal makes coverage a per-row property. Advanced only over probes that started with a live deadline. A probe against a spent deadline is charged a read but never opens the repo, so counting it would step the window over candidates nobody examined. Persisted from inside sweep_pass, next to the row cursor. The run-end site never executes: spawn_legacy_cid_sweep drops the future on SIGTERM. Traversal state is owned by the wrapper because the dead-read cap can pause a run mid-traversal and a later run ends it. The sweep re-arms on a timer instead of running once per boot, so coverage is wall-clock rather than a reboot count, and returns only on a failing pass query. The unconditional sleep is what keeps a repaired table from spinning. Verified by mutation: no rotation strands the holder; an offset continuation shifts the window; advancing to the window end both skips an unread holder and burns an unprobed window; per-pass advancement drifts one row's window from another's; no re-arm leaves the holder unreached; a run-scoped accumulator applies the wrong arm after a cap pause; and deleting the v26 entry fails the upgrade path. --- crates/gitlawb-node/src/db/mod.rs | 202 +++++ crates/gitlawb-node/src/ipfs_pin.rs | 281 +++++- crates/gitlawb-node/src/main.rs | 20 +- crates/gitlawb-node/src/test_support.rs | 1052 +++++++++++++++++++++++ 4 files changed, 1529 insertions(+), 26 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index cee541cd..d2b6206d 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1088,6 +1088,41 @@ const MIGRATIONS: &[Migration] = &[ "CREATE INDEX IF NOT EXISTS idx_repos_created_at_id ON repos (created_at ASC, id ASC)", ], }, + Migration { + version: 26, + name: "pin_repair_sweep_discovery_cursor", + stmts: &[ + // #173 round 13 (F5): discovery probes at most + // `MAX_LEGACY_DISCOVERY_PROBES` warm candidates per source-less row, taken + // from a list ordered `(created_at, id)`. That order is stable, so without a + // continuation every traversal probed the same oldest sixteen and a holder + // at position seventeen was never reached by anything, on any node, ever. + // These two columns are the boundary the next traversal's window starts + // after, so coverage becomes a bounded number of traversals rather than + // unreachable. + // + // STEERABILITY is why this is a keyset KEY and not an offset into the list. + // `repo_id` derives from a grindable owner DID, so the one thing an attacker + // must not be able to do is move the window off the true holder. Candidates + // enter and leave the warm list between traversals (a cold repo warming on a + // Tigris-backed node, a fresh registration, a deletion), and every such + // change silently renumbers an offset while leaving a key's boundary exactly + // where it was. Fresh registrations sort LAST under `created_at` and cannot + // be backdated, so they can only ever be appended behind the window. + // + // RESIDUAL, stated rather than implied: an operator who can insert repos + // with an arbitrary `created_at` can still place candidates between the + // continuation and the holder and delay it by a traversal per sixteen rows + // inserted. That is a privileged write, it costs a real repo row each, and + // it delays rather than prevents, since the window keeps advancing. + // + // NEW versioned migration (never appended to an applied block, INV-7). NOT + // NULL DEFAULT '' so an existing `pin_repair_sweep` row reads as "start at + // the head of the list", which is the same thing a never-swept node reads. + "ALTER TABLE pin_repair_sweep ADD COLUMN IF NOT EXISTS discovery_cursor_created_at TEXT NOT NULL DEFAULT ''", + "ALTER TABLE pin_repair_sweep ADD COLUMN IF NOT EXISTS discovery_cursor_id TEXT NOT NULL DEFAULT ''", + ], + }, ]; /// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). @@ -2878,6 +2913,67 @@ impl Db { Ok(()) } + /// Where the sweep's DISCOVERY window left off, as a `(created_at, id)` keyset + /// key, or `("", "")` before any traversal has completed one. + /// + /// A second, independent position from [`Db::pin_repair_cursor`]: that one walks + /// `pinned_cids` rows, this one walks the warm CANDIDATE list a source-less row is + /// probed against. Both are per-TRAVERSAL, and the candidate one only ever moves + /// at the end of a completed traversal, so every pass of one traversal reads the + /// same value and every source-less row in it shares one window. + /// + /// A key rather than an offset. Repos enter and leave the warm candidate list + /// between traversals (a cold repo warming on a Tigris-backed node, a fresh + /// registration, a deletion), and an offset silently means a different candidate + /// once anything below it moves, which slides the window off the row it was about + /// to reach. A key names the boundary itself, so an insert below it is invisible. + /// The key is the RAW stored `created_at` text (`ScanRepoRow::created_at_key`), + /// never a re-serialized `DateTime`, for the reason that struct's own doc gives. + pub async fn discovery_continuation(&self) -> Result<(String, String)> { + let row = sqlx::query( + "SELECT discovery_cursor_created_at, discovery_cursor_id + FROM pin_repair_sweep WHERE id = 1", + ) + .fetch_optional(&self.pool) + .await?; + Ok(row + .map(|r| { + ( + r.get::("discovery_cursor_created_at"), + r.get::("discovery_cursor_id"), + ) + }) + .unwrap_or_default()) + } + + /// Persist the discovery window's continuation at the end of a completed traversal. + /// + /// The INSERT arm names `cursor` explicitly with `''`. v23 declares that column + /// `NOT NULL` and seeds NO row, so a never-swept node has nothing to update and an + /// upsert naming only the continuation columns would fail its NOT NULL check. + /// Every caller treats a failed persist as warn-only, so that failure would be + /// SILENT and the window would never rotate on exactly the nodes this sweep exists + /// for. `''` is the same value `pin_repair_cursor` reads as "never swept", so + /// seeding it here starts no walk anywhere but the top of the table. + /// + /// The UPDATE arm touches ONLY the two continuation columns. Writing `cursor` there + /// too would clobber a live row-walk position with `''` every time the window + /// rotated, rewinding the `pinned_cids` walk to the start of the table. + pub async fn set_discovery_continuation(&self, created_at_key: &str, id: &str) -> Result<()> { + sqlx::query( + "INSERT INTO pin_repair_sweep (id, cursor, discovery_cursor_created_at, discovery_cursor_id) + VALUES (1, '', $1, $2) + ON CONFLICT (id) DO UPDATE SET + discovery_cursor_created_at = EXCLUDED.discovery_cursor_created_at, + discovery_cursor_id = EXCLUDED.discovery_cursor_id", + ) + .bind(created_at_key) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + /// The repository a pinned object was recorded from (`pinned_cids.repo_id`), /// or `None` for a legacy pin (recorded before provenance existed) or an /// unpinned oid. `GET /ipfs/{cid}` uses this to gate+serve the ONE source @@ -6697,6 +6793,112 @@ mod ref_certificate_tests { ); } + /// U4 (#173 round 13, F5, INV-7 upgrade path): an existing node past v1 gets the + /// discovery-continuation columns from its OWN v26 entry, proven by dropping the + /// columns plus their `schema_migrations` row and re-running the real migration + /// code. + /// + /// The round-trip runs on a NEVER-SWEPT database, with no `pin_repair_sweep` row at + /// all, because that is the state the setter's insert arm is written for. v23 + /// declares `cursor` NOT NULL and seeds no row, so an upsert naming only the two new + /// columns fails its NOT NULL check on exactly the nodes this sweep exists for, and + /// every caller of the setter treats a failure as warn-only, so the window would + /// simply never rotate and nothing would say so. Asserting the read-back is what + /// makes that failure visible here. + /// + /// MUTATION (RED): delete the v26 entry from `MIGRATIONS` and the fresh-chain + /// round-trip fails on the missing columns. + #[sqlx::test] + async fn v26_discovery_continuation_applies_on_upgrade(pool: PgPool) { + async fn continuation_columns_exist(pool: &PgPool) -> bool { + sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM information_schema.columns + WHERE table_name = 'pin_repair_sweep' + AND column_name IN ('discovery_cursor_created_at', 'discovery_cursor_id')", + ) + .fetch_one(pool) + .await + .unwrap() + == 2 + } + + let db = Db::for_testing(pool.clone()); + db.run_migrations().await.unwrap(); + assert!( + continuation_columns_exist(&pool).await, + "the fresh migration chain must carry the discovery continuation columns" + ); + + // Simulate a node at pre-v26: drop the columns and their migration record. + sqlx::query( + "ALTER TABLE pin_repair_sweep + DROP COLUMN IF EXISTS discovery_cursor_created_at, + DROP COLUMN IF EXISTS discovery_cursor_id", + ) + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 26") + .execute(&pool) + .await + .unwrap(); + assert!( + !continuation_columns_exist(&pool).await, + "precondition: columns and their migration record removed" + ); + + db.run_migrations().await.unwrap(); + assert!( + continuation_columns_exist(&pool).await, + "v26 must add the continuation columns on an upgrading node" + ); + + // NEVER SWEPT: no `pin_repair_sweep` row exists, so the setter has to INSERT and + // its insert arm has to satisfy v23's NOT NULL `cursor`. + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT count(*) FROM pin_repair_sweep") + .fetch_one(&pool) + .await + .unwrap(), + 0, + "precondition: the sweep has never run on this node" + ); + assert_eq!( + db.discovery_continuation().await.unwrap(), + (String::new(), String::new()), + "an unswept node reads the empty continuation, which means the head of the list" + ); + db.set_discovery_continuation("2020-01-01T00:00:00+00:00", "repo-42") + .await + .expect("the continuation persists on a never-swept node"); + assert_eq!( + db.discovery_continuation().await.unwrap(), + ( + "2020-01-01T00:00:00+00:00".to_string(), + "repo-42".to_string() + ), + "the continuation round-trips" + ); + assert_eq!( + db.pin_repair_cursor().await.unwrap(), + "", + "the insert arm seeds the row-walk cursor at the head of the table" + ); + + // A rotation must never move the row walk. Park the row cursor, rotate again, + // and read it back. + db.set_pin_repair_cursor("ff00").await.unwrap(); + db.set_discovery_continuation("2021-06-01T00:00:00+00:00", "repo-99") + .await + .unwrap(); + assert_eq!( + db.pin_repair_cursor().await.unwrap(), + "ff00", + "the update arm touches only the continuation columns, so an in-progress \ + table walk is never rewound by a window rotation" + ); + } + /// INV-7: upgrade-path test — seed a database at v9 with duplicate /// ref_certificates, then let the real v10 migration fire via /// run_migrations(). This exercises the migration code path rather than diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index c754d051..2fb53cfe 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -302,6 +302,28 @@ pub(crate) struct SweepStats { /// [`DISCOVERY_ROW_BUDGET_DIVISOR`]). Reporting only, like `retryable_skips`: it /// drives no control decision, it just keeps a starved pass from being silent. pub discovery_budget_spent: bool, + /// Why the run stopped. Meaningful on a RUN (`sweep_legacy_provider_cids` and the + /// re-arm wrapper); on a single pass it is always `Completed` and says nothing. + pub stop: SweepStop, +} + +/// Why a sweep run ended, which is what [`run_sweep_rearmed`] dispatches on. +/// +/// Two of the three arms are re-armable and one is not. A run that walked to the end of +/// the table and a run that paused on [`MAX_DEAD_ROW_READS_PER_RUN`] both left the node +/// in a state a later run improves, so the wrapper sleeps and goes again. A failing pass +/// QUERY is a broken database, and retrying it on a timer would turn one logged failure +/// into an endless stream of them, so the wrapper returns and leaves the run one-shot, +/// exactly as it was before the re-arm existed. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SweepStop { + /// The ordered walk reached the end of the table (a short batch). + #[default] + Completed, + /// Enough fruitless reads for one run; the cursor stays mid-table. + PausedOnDeadReadCap, + /// A pass's batch query or cursor write failed. + PassFailed, } /// How many fruitless object reads one sweep run will spend before it stops and leaves @@ -346,9 +368,16 @@ const DISCOVERY_ROW_BUDGET_DIVISOR: u32 = 4; /// filter runs ONCE here rather than per row, on the blocking pool, because O(repos) /// stat calls per row would park a tokio worker for the whole boot sweep. struct DiscoveryCtx { - /// Warm candidates with their validated disk paths, oldest-first by - /// `(created_at, id)`. - candidates: Vec<(crate::db::RepoRecord, std::path::PathBuf)>, + /// Warm candidates with their RAW `(created_at, id)` keyset key and validated disk + /// path, ROTATED so the traversal's window starts at the head. + /// + /// The key is `ScanRepoRow::created_at_key`, the stored text, carried through rather + /// than re-derived from `RepoRecord::created_at`: re-serializing the parsed + /// `DateTime` is not guaranteed to reproduce the stored bytes (that struct says so + /// itself), and the keyset comparison this feeds is a TEXT comparison against the + /// SQL order, so a key off by one character rotates the list to a boundary the query + /// never had. + candidates: Vec<(crate::db::RepoRecord, String, std::path::PathBuf)>, /// The ceiling on the whole pass's discovery, so one pass costs at most one /// `git_timeout` in total on top of the per-row probe cap. Per PASS, not per run: /// `load_discovery_ctx` runs once per `sweep_pass` and a run loops passes. @@ -359,6 +388,74 @@ struct DiscoveryCtx { pass_deadline: Instant, } +/// What one TRAVERSAL of the `pinned_cids` table learned about how far its discovery +/// window actually got, and therefore where the next traversal's window may start. +/// +/// Owned by [`run_sweep_rearmed`] and passed `&mut` through every run and every pass of +/// the traversal, which is the whole point of the type: the dead-read cap can PAUSE a +/// run in the middle of a traversal, and the run that later reaches the short batch is a +/// different run. Rebuilding this per run means that final run sees an empty accumulator +/// and applies the hold arm (or the reset arm) for windows the earlier run really did +/// probe, so the traversal advances by nothing and the sweep stalls on the same window +/// forever. Its lifetime is the traversal, so that is what it is scoped to. +#[derive(Debug, Default)] +pub(crate) struct DiscoveryTraversalState { + /// The `(created_at_key, id)` of the last candidate whose probe STARTED with the + /// row's deadline still live. + /// + /// A probe started against a dead deadline is charged a read (the U3 boundary row is + /// exactly this) but learns nothing: `db_bounded` returns immediately and the + /// candidate is left unread. Advancing over one would skip a candidate nobody looked + /// at, which is the same hole the continuation exists to close, one window narrower. + last_live_probe: Option<(String, String)>, + /// A row reached the probe cap with candidates still unprobed AND spent at least one + /// live-budget probe doing it. This is the arm that ADVANCES: there is a next window + /// and the traversal earned the right to move to it. + cap_exhausted_with_budget: bool, + /// The whole warm list fit inside one window, observed by a row with live budget. + /// There is no next window, so the continuation RESETS: leaving a stale key behind + /// on a list that has since shrunk below it would rotate every later traversal to an + /// empty tail and then wrap to the same prefix forever. + fit_under_cap: bool, +} + +impl DiscoveryTraversalState { + /// The advance to apply at the end of a completed traversal, or `None` to hold the + /// continuation where it is. + /// + /// Three arms, in this order. ADVANCE when a row ran out of window with budget left + /// to spend, to the last candidate actually read live. RESET when the list fit under + /// the cap, because there is nothing past the window to advance to. HOLD otherwise, + /// which is the starved traversal: nothing was probed live, so burning a window + /// would skip candidates on the strength of reads that never happened. + fn advance(&self) -> Option<(String, String)> { + if self.cap_exhausted_with_budget { + return self.last_live_probe.clone(); + } + if self.fit_under_cap { + return Some((String::new(), String::new())); + } + None + } + + /// Fold one finished row's window observation in. + /// + /// A row that read NOTHING with live budget contributes nothing at all, neither arm. + /// Such a row is evidence about the clock, not about the candidates: letting it set + /// either flag would move or reset the window on the strength of probes that were + /// charged but never made. + fn note_row(&mut self, live_probes: usize, fits_under_cap: bool) { + if live_probes == 0 { + return; + } + if fits_under_cap { + self.fit_under_cap = true; + } else { + self.cap_exhausted_with_budget = true; + } + } +} + /// Build one pass's discovery candidate list. /// /// Three filters, all applied before any probe so a rejected candidate costs nothing @@ -405,7 +502,7 @@ async fn load_discovery_ctx( // result is reused for every source-less row, so the paging cost is paid once. let page_rows = crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS; let mut cursor: Option<(String, String)> = None; - let mut candidates: Vec = Vec::new(); + let mut candidates: Vec<(crate::db::RepoRecord, String)> = Vec::new(); loop { let page = db .list_repos_page_for_scan( @@ -418,7 +515,11 @@ async fn load_discovery_ctx( let Some(last) = page.last() else { break }; cursor = Some((last.created_at_key.clone(), last.repo.id.clone())); let last_page = page.len() < page_rows; - candidates.extend(page.into_iter().filter(|r| !r.quarantined).map(|r| r.repo)); + candidates.extend( + page.into_iter() + .filter(|r| !r.quarantined) + .map(|r| (r.repo, r.created_at_key)), + ); if last_page { break; } @@ -428,13 +529,13 @@ async fn load_discovery_ctx( let warm = tokio::task::spawn_blocking(move || { candidates .into_iter() - .filter_map(|repo| { + .filter_map(|(repo, created_at_key)| { match crate::git::repo_store::validated_repo_disk_path( &repos_dir, &repo.owner_did, &repo.name, ) { - Ok(p) if p.is_dir() => Some((repo, p)), + Ok(p) if p.is_dir() => Some((repo, created_at_key, p)), // Cold: not on this node's disk right now. It is not evidence about // any row (see `discover_legacy_row`), so it is simply absent here. Ok(_) => None, @@ -447,6 +548,31 @@ async fn load_discovery_ctx( .collect::>() }) .await?; + let mut warm = warm; + + // ROTATE to the traversal's window. The list is already in `(created_at, id)` order, + // so the window is the first `MAX_LEGACY_DISCOVERY_PROBES` entries strictly after the + // persisted continuation, wrapping through the prefix when it runs off the end. + // + // After the warm filter, deliberately: a cold or quarantined candidate is not a + // window slot the traversal spent, so rotating first would let a node full of cold + // repos advance the continuation past warm candidates nobody ever probed. The window + // is sixteen WARM candidates. + // + // Every pass of a traversal reads the same persisted value (it only moves in the + // traversal-ending pass), so the window is stable across the traversal by + // construction and two source-less rows in different passes probe the same repos. + let (cont_created_at, cont_id) = db.discovery_continuation().await?; + if !cont_created_at.is_empty() || !cont_id.is_empty() { + let split = warm + .iter() + .position(|(repo, created_at_key, _)| { + (created_at_key.as_str(), repo.id.as_str()) + > (cont_created_at.as_str(), cont_id.as_str()) + }) + .unwrap_or(warm.len()); + warm.rotate_left(split); + } Ok(DiscoveryCtx { candidates: warm, @@ -497,6 +623,7 @@ async fn discover_legacy_row( git_bin: &str, git_timeout: Duration, db: &crate::db::Db, + traversal: &mut DiscoveryTraversalState, ) -> (DiscoveryOutcome, usize) { let mut reads = 0usize; if ctx.is_none() { @@ -528,10 +655,30 @@ async fn discover_legacy_row( ); let mut retryable = false; + // How many of this row's probes actually STARTED with budget to spend, and whether + // the whole warm list fits in one window. Together they pick the traversal's advance + // arm once the row is done. + let mut live_probes = 0usize; + let fits_under_cap = ctx.candidates.len() <= MAX_LEGACY_DISCOVERY_PROBES; // Every candidate that gets this far is READ, so taking the first // MAX_LEGACY_DISCOVERY_PROBES bounds the expensive work exactly. Candidates the // filters already rejected never reach here and so cost nothing against the cap. - for (repo, repo_path) in ctx.candidates.iter().take(MAX_LEGACY_DISCOVERY_PROBES) { + for (repo, created_at_key, repo_path) in ctx.candidates.iter().take(MAX_LEGACY_DISCOVERY_PROBES) + { + // The live-budget test, taken BEFORE the probe and against the SAME deadline the + // probe is handed. Two shapes reach a probe with the deadline already gone and + // both are charged a read for it: U3's boundary row, admitted by a skip guard of + // `now >= pass_deadline` with a sliver of budget that `row_deadline` clamps to + // nothing, and every candidate queued behind a wedged one inside a row. In both, + // `db_bounded` returns on the spent deadline and the repo is never opened. They + // are reads, not looks, and the continuation must not advance over them: doing so + // skips candidates nobody examined, which is the hole the continuation exists to + // close, one window narrower. + let live = Instant::now() < row_deadline; + if live { + live_probes += 1; + traversal.last_live_probe = Some((created_at_key.clone(), repo.id.clone())); + } // Counted before the match, because the read is spent whatever it returns. This // is the quantity the caller charges against the per-run budget. reads += 1; @@ -557,6 +704,7 @@ async fn discover_legacy_row( if let Err(e) = db.mark_pin_sources_incomplete(sha, "").await { tracing::warn!(sha = %sha, err = %e, "sweep discovery: failed to mark the pin-source set incomplete"); } + traversal.note_row(live_probes, fits_under_cap); return (DiscoveryOutcome::Repaired, reads); } // The bytes could not be read from this WARM candidate right now, which IS @@ -570,6 +718,7 @@ async fn discover_legacy_row( } } } + traversal.note_row(live_probes, fits_under_cap); if ctx.candidates.len() > MAX_LEGACY_DISCOVERY_PROBES { // Cap exhausted with candidates left unprobed: RETRYABLE, never terminal. The // probe order is deterministic, but "a re-walk finds the same nothing" only @@ -603,6 +752,7 @@ async fn sweep_pass( git_timeout: Duration, batch: i64, db: &crate::db::Db, + traversal: &mut DiscoveryTraversalState, ) -> Result { let cursor = db.pin_repair_cursor().await?; let rows = db.pinned_cids_after(&cursor, batch).await?; @@ -649,9 +799,16 @@ async fn sweep_pass( // what makes an unrepairable row COST something rather than just being skipped. let mut row_read_attempted = false; if sources.is_empty() { - let (outcome, reads) = - discover_legacy_row(&sha, &mut discovery, repos_dir, git_bin, git_timeout, db) - .await; + let (outcome, reads) = discover_legacy_row( + &sha, + &mut discovery, + repos_dir, + git_bin, + git_timeout, + db, + traversal, + ) + .await; match outcome { DiscoveryOutcome::Repaired => { repaired += 1; @@ -759,6 +916,22 @@ async fn sweep_pass( } db.set_pin_repair_cursor(&last).await?; + // A short batch is the end of the table, so this pass ended the TRAVERSAL: apply the + // window advance the traversal earned, then start a fresh accumulator for the next + // one. Persisting from HERE, not from the end of the run, is what survives the + // shutdown `select!` in `spawn_legacy_cid_sweep`: a drop mid-traversal loses only + // the accumulator, so the next traversal repeats a window rather than skipping one. + // + // The write is warn-only. A failed persist leaves the old continuation, and the next + // traversal probes the same window again, which is wasted work and never a gap. + if (scanned as i64) < batch { + if let Some((created_at_key, id)) = traversal.advance() { + if let Err(e) = db.set_discovery_continuation(&created_at_key, &id).await { + tracing::warn!(err = %e, "failed to persist the sweep discovery continuation"); + } + } + *traversal = DiscoveryTraversalState::default(); + } Ok(SweepStats { scanned, repaired, @@ -766,6 +939,7 @@ async fn sweep_pass( retryable_skips, dead_row_reads, discovery_budget_spent, + stop: SweepStop::Completed, }) } @@ -778,8 +952,9 @@ pub(crate) async fn sweep_legacy_provider_cids_once( git_timeout: Duration, batch: i64, db: &crate::db::Db, + traversal: &mut DiscoveryTraversalState, ) -> Result { - sweep_pass(repos_dir, git_bin, git_timeout, batch, db).await + sweep_pass(repos_dir, git_bin, git_timeout, batch, db, traversal).await } /// U4 (#173): the one-shot legacy provider-CID migration sweep. @@ -839,14 +1014,16 @@ pub(crate) async fn sweep_legacy_provider_cids( batch: i64, delay: Duration, db: &crate::db::Db, + traversal: &mut DiscoveryTraversalState, ) -> SweepStats { let mut totals = SweepStats::default(); let mut completed = false; loop { - let pass = match sweep_pass(repos_dir, git_bin, git_timeout, batch, db).await { + let pass = match sweep_pass(repos_dir, git_bin, git_timeout, batch, db, traversal).await { Ok(p) => p, Err(e) => { tracing::warn!(err = %e, "legacy provider-CID sweep pass failed; stopping"); + totals.stop = SweepStop::PassFailed; break; } }; @@ -860,6 +1037,7 @@ pub(crate) async fn sweep_legacy_provider_cids( // rather than after an extra empty pass, and do NOT sleep on the way out. if (pass.scanned as i64) < batch { completed = true; + totals.stop = SweepStop::Completed; break; } // Enough fruitless reads for one run. Stop WITHOUT completing, so the cursor @@ -871,6 +1049,7 @@ pub(crate) async fn sweep_legacy_provider_cids( dead_row_reads = totals.dead_row_reads, "legacy provider-CID sweep pausing: too many unrepairable rows this run" ); + totals.stop = SweepStop::PausedOnDeadReadCap; break; } tokio::time::sleep(delay).await; @@ -890,6 +1069,82 @@ pub(crate) async fn sweep_legacy_provider_cids( totals } +/// How long the sweep waits between runs before walking the table again. +/// +/// Coverage of the discovery window is per TRAVERSAL, and a node with more warm repos +/// than one window needs several of them, so how long a source-less row waits for its +/// holder's window is set by how often traversals happen. Tying that to reboots would +/// make it a reboot count on a node that never reboots, which is the healthy node. +/// +/// Five minutes is chosen against what a run COSTS on a settled node, not against how +/// fast the migration should finish: a fully repaired table is one indexed range scan +/// per batch and a codec decode per row, no object reads at all, so the standing cost is +/// a few queries every five minutes and the migration still converges in hours rather +/// than never. It is also the anti-hot-spin floor for the degenerate case, an empty or +/// fully repaired table where a run returns immediately. +pub(crate) const SWEEP_REARM_DELAY: Duration = Duration::from_secs(300); + +/// Run the legacy provider-CID sweep on a timer until shutdown or a broken database. +/// +/// Owns the [`DiscoveryTraversalState`] across runs, which is the reason this is a +/// wrapper and not a loop inside `sweep_legacy_provider_cids`: a run can PAUSE +/// mid-traversal on [`MAX_DEAD_ROW_READS_PER_RUN`], and the traversal it was in is +/// finished by a later run, which has to apply the advance the earlier run earned. +/// +/// Sleeps `rearm_delay` after EVERY re-armable run, unconditionally. Not conditional on +/// the run having done work: a run over an empty or fully repaired table returns +/// immediately, and without the sleep this loop would spin the database as fast as it +/// can answer. +/// +/// Returns only on [`SweepStop::PassFailed`], preserving the one-shot behavior a failing +/// database had before the re-arm existed. On a healthy node it never returns, which is +/// why the per-run summary is logged HERE rather than by the caller off the awaited +/// value. +pub(crate) async fn run_sweep_rearmed( + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + batch: i64, + delay: Duration, + rearm_delay: Duration, + db: &crate::db::Db, +) -> SweepStats { + let mut totals = SweepStats::default(); + let mut traversal = DiscoveryTraversalState::default(); + loop { + let run = sweep_legacy_provider_cids( + repos_dir, + git_bin, + git_timeout, + batch, + delay, + db, + &mut traversal, + ) + .await; + if run.repaired > 0 { + tracing::info!( + scanned = run.scanned, + repaired = run.repaired, + passes = run.passes, + stop = ?run.stop, + "legacy provider-CID sweep run finished" + ); + } + totals.scanned += run.scanned; + totals.repaired += run.repaired; + totals.passes += run.passes; + totals.retryable_skips += run.retryable_skips; + totals.dead_row_reads += run.dead_row_reads; + totals.discovery_budget_spent |= run.discovery_budget_spent; + totals.stop = run.stop; + if run.stop == SweepStop::PassFailed { + return totals; + } + tokio::time::sleep(rearm_delay).await; + } +} + // Test-only cost-gate counter (R8, U7): how many times the opportunistic repair // read an object's bytes on the skip path. The codec gate must spare a CIDv1/raw // row this read; the counter is the both-ways guard (removing the gate reads the diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 9d140fbd..4acbb9bc 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -700,7 +700,7 @@ async fn main() -> Result<()> { Ok(()) } -/// U4 (#173): spawn the one-shot legacy provider-CID repair sweep. Releases before this +/// U4 (#173): spawn the periodic legacy provider-CID repair sweep. Releases before this /// version stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`, /// and this version's `/ipfs/{cid}` resolver withholds any row whose stored key is not /// the raw-content CID. The opportunistic repair on the pin path only fires when a push @@ -708,7 +708,8 @@ async fn main() -> Result<()> { /// need a walk. DETACHED, never on the boot path: the caller keeps serving while this /// runs, and the sweep's own batch bound plus inter-batch delay keep it off the DB's /// critical path. Its cursor is durable, so a restart mid-walk resumes instead of -/// rewinding. +/// rewinding. It re-arms on `SWEEP_REARM_DELAY` and so returns only on a failing pass +/// query, which is why the awaited value is no longer worth logging here. /// /// A named function rather than an inline block in `main` so the WIRING has a seam a /// test can call: that the task is spawned at all, that it reads its batch and delay @@ -725,17 +726,10 @@ fn spawn_legacy_cid_sweep(state: &AppState, config: &Config) -> tokio::task::Joi let mut shutdown_rx = state.subscribe_shutdown(); tokio::spawn(async move { tokio::select! { - stats = ipfs_pin::sweep_legacy_provider_cids( - &repos_dir, &git_bin, git_timeout, batch, delay, &db, - ) => { - if stats.repaired > 0 { - tracing::info!( - scanned = stats.scanned, - repaired = stats.repaired, - "legacy provider-CID sweep finished" - ); - } - } + _ = ipfs_pin::run_sweep_rearmed( + &repos_dir, &git_bin, git_timeout, batch, delay, + ipfs_pin::SWEEP_REARM_DELAY, &db, + ) => {} // Shutdown mid-walk simply drops the run; the persisted cursor means the // next boot picks up where this one stopped. _ = shutdown_rx.changed() => {} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 0dcef1a0..c2bb0884 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -6481,6 +6481,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ) .await; assert_eq!(stats.repaired, 1, "the sweep repairs the one legacy row"); @@ -6558,6 +6559,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ) .await; assert_eq!(stats.repaired, 0, "an unrepairable row is not repaired"); @@ -6609,6 +6611,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ) .await; assert_eq!(stats.scanned, 1, "the sweep walked the row"); @@ -6662,6 +6665,7 @@ mod tests { std::time::Duration::from_secs(state.config.git_service_timeout_secs), 2, &state.db, + &mut Default::default(), ) .await .expect("one pass runs"); @@ -6717,6 +6721,7 @@ mod tests { git_timeout, 2, &state.db, + &mut Default::default(), ) .await .expect("pass 1 runs"); @@ -6730,6 +6735,7 @@ mod tests { git_timeout, 2, &state.db, + &mut Default::default(), ) .await .expect("pass 2 runs"); @@ -6806,6 +6812,7 @@ mod tests { 1, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -6870,6 +6877,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -6892,6 +6900,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -6968,6 +6977,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -6988,6 +6998,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -7047,6 +7058,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -7076,6 +7088,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -7156,6 +7169,7 @@ mod tests { batch, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -7252,6 +7266,7 @@ mod tests { 1, std::time::Duration::from_millis(300), &state.db, + &mut Default::default(), ), ) .await @@ -7313,6 +7328,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -7382,6 +7398,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ) .await; ticker.abort(); @@ -7417,6 +7434,7 @@ mod tests { 4, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -7447,6 +7465,7 @@ mod tests { 4, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -7498,6 +7517,7 @@ mod tests { 2, delay, &state.db, + &mut Default::default(), ) .await; let elapsed = started.elapsed(); @@ -7599,6 +7619,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -7667,6 +7688,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -7685,6 +7707,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -7735,6 +7758,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -7820,6 +7844,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -7867,6 +7892,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -7954,6 +7980,7 @@ mod tests { batch, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -8038,6 +8065,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -8102,6 +8130,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -8204,6 +8233,7 @@ mod tests { std::time::Duration::from_secs(4), 16, &state.db, + &mut Default::default(), ), ) .await @@ -8290,6 +8320,7 @@ mod tests { std::time::Duration::from_secs(4), 16, &state.db, + &mut Default::default(), ), ) .await @@ -8324,6 +8355,1016 @@ mod tests { ); } + // ---- #173 round 13, F5: the per-traversal discovery window continuation ---- + + /// A repo row at a chosen point in the sweep's `(created_at, id)` candidate order, + /// `pos` seconds past a fixed base so the order is the fixture's to set rather than + /// the clock's. Negative positions sort BELOW the base, which is how a fixture models + /// a candidate entering the warm list underneath an already-persisted continuation. + fn seed_repo_at(owner_did: &str, name: &str, pos: i64) -> RepoRecord { + let created_at = chrono::DateTime::parse_from_rfc3339("2020-01-01T12:00:00Z") + .expect("the fixture base parses") + .with_timezone(&Utc) + + chrono::Duration::seconds(pos); + RepoRecord { + created_at, + updated_at: created_at, + ..seed_repo(owner_did, name) + } + } + + /// The keyset key the sweep stores for a candidate: the RAW `created_at` text as + /// `create_repo` wrote it, plus the repo id. + fn candidate_key(repo: &RepoRecord) -> (String, String) { + (repo.created_at.to_rfc3339(), repo.id.clone()) + } + + /// Seed `n` warm candidates in candidate order (position 1 is the oldest). The one at + /// 1-based `holder` is the already-cloned bare named there and really holds the + /// fixture's objects; every other position is an empty bare that holds nothing, so a + /// probe against it costs a real object read and finds nothing. + async fn seed_candidate_ladder( + db: &crate::db::Db, + owner_did: &str, + slug: &str, + prefix: &str, + n: usize, + holder: Option<(usize, &str)>, + ) -> Vec { + let mut rows = Vec::new(); + for pos in 1..=n { + let name = match holder { + Some((hp, hn)) if hp == pos => hn.to_string(), + _ => { + let name = format!("{prefix}{pos}"); + init_empty_bare( + &std::path::PathBuf::from("/tmp") + .join(slug) + .join(format!("{name}.git")), + ); + name + } + }; + let repo = seed_repo_at(owner_did, &name, pos as i64); + db.create_repo(&repo).await.expect("seed candidate"); + rows.push(repo); + } + rows + } + + /// Copy ONE blob between SHA-256 bares, preserving its oid. A bare clone carries + /// every object in the fixture, and the cross-batch scenario needs a candidate that + /// holds exactly one of them. + fn copy_blob_into_bare(src: &std::path::Path, dst: &std::path::Path, oid: &str) { + use std::io::Write; + use std::process::{Command, Stdio}; + let blob = Command::new("git") + .args(["cat-file", "blob", oid]) + .current_dir(src) + .output() + .expect("git runs"); + assert!(blob.status.success(), "cat-file blob {oid}"); + let mut child = Command::new("git") + .args(["hash-object", "-w", "-t", "blob", "--stdin"]) + .current_dir(dst) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .expect("git runs"); + child + .stdin + .take() + .expect("stdin") + .write_all(&blob.stdout) + .expect("feed the blob"); + let out = child.wait_with_output().expect("hash-object finishes"); + assert!(out.status.success(), "hash-object -w"); + assert_eq!( + String::from_utf8_lossy(&out.stdout).trim(), + oid, + "the copied blob keeps its oid, or the fixture is not the object the row names" + ); + } + + /// Poll `f` until it yields a value or `limit` runs out. Several scenarios drive the + /// re-arm wrapper, which on a healthy table never returns, so the assertion has to be + /// on DB state observed while it runs. + async fn poll_until(limit: std::time::Duration, mut f: F) -> Option + where + F: FnMut() -> Fut, + Fut: std::future::Future>, + { + let deadline = std::time::Instant::now() + limit; + loop { + if let Some(v) = f().await { + return Some(v); + } + if std::time::Instant::now() >= deadline { + return None; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + } + + /// F5 scenario 1 (#173 round 13): a holder past the probe cap is REACHED. + /// + /// `discover_legacy_row` probes the first `MAX_LEGACY_DISCOVERY_PROBES` of a list + /// ordered `(created_at, id)`. That order is stable and the list was rebuilt from + /// scratch every run, so before the continuation every traversal on every node probed + /// the same oldest sixteen and a holder at position seventeen was unreachable by + /// anything: not a later pass, not a later run, not a reboot. Seventeen warm + /// candidates with only the newest holding the object; the first traversal must + /// repair nothing and persist where it got to, the second must start after that and + /// repair the row, and neither may exceed the probe cap. + /// + /// RED before the rotation: both traversals probe the same first sixteen and the row + /// keeps its provider key forever. + #[sqlx::test] + async fn sweep_discovery_rotation_reaches_later_candidates(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + // `rotsrc` carries the bytes but has NO repos row, so it is never a candidate; + // `rotheld` is the candidate that really holds them. + let fx = seed_cid_repos(&slug, &short, &["rotsrc", "rotheld"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("rotsrc.git"); + let candidates = seed_candidate_ladder( + &state.db, + &owner_did, + &slug, + "rotcand", + crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES + 1, + Some((crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES + 1, "rotheld")), + ) + .await; + let (raw_cid, provider_cid) = seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; + + let first = tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the first traversal terminates"); + + assert_eq!( + first.repaired, 0, + "the holder sits past the probe cap, so the first window cannot reach it" + ); + assert_eq!( + first.dead_row_reads, + crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES, + "the first traversal spends exactly one window of probes on the row" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "the row still carries its legacy provider key after the first traversal" + ); + assert_eq!( + state.db.discovery_continuation().await.unwrap(), + candidate_key(&candidates[crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES - 1]), + "a completed traversal that ran out of window persists the last candidate it \ + actually read, so the next one starts after it instead of repeating it" + ); + + let second = tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the second traversal terminates"); + + assert_eq!( + second.repaired, 1, + "the second traversal's window starts at the seventeenth candidate and \ + repairs the row" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the key is rewritten to the raw-content CID" + ); + assert!( + second.dead_row_reads <= crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES, + "the rotation moves the window, it does not widen it: {} reads in one \ + traversal", + second.dead_row_reads + ); + assert_ne!(raw_cid, provider_cid, "control: the two keys really differ"); + } + + /// F5 scenario 2 (#173 round 13, MUST-NOT, the steerability negative): candidates + /// appearing between traversals must not move the window off the holder. + /// + /// The continuation is a keyset KEY, not an offset, and this is the difference. + /// Freshly registered repos sort LAST under `created_at` and cannot be backdated, so + /// they can only ever land behind the window. Candidates can also enter BELOW the + /// continuation without any mint at all: a cold repo warms on a Tigris-backed node, + /// an operator restores an archived one. Every one of those silently renumbers an + /// offset, and sixteen of them slide an offset window clean off the candidate it was + /// about to reach, while a key names the boundary itself and does not care what + /// appeared underneath it. + /// + /// RED under an offset continuation: the second traversal's window starts sixteen + /// entries into a list that grew underneath it and never reaches the holder. + #[sqlx::test] + async fn sweep_discovery_rotation_survives_minted_candidates(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["mintsrc", "mintheld"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("mintsrc.git"); + let candidates = seed_candidate_ladder( + &state.db, + &owner_did, + &slug, + "mintcand", + cap + 1, + Some((cap + 1, "mintheld")), + ) + .await; + let (raw_cid, provider_cid) = seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; + + tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the first traversal terminates"); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "precondition: the first window does not reach the holder" + ); + let boundary = state.db.discovery_continuation().await.unwrap(); + assert_eq!( + boundary, + candidate_key(&candidates[cap - 1]), + "precondition: the window boundary is the sixteenth candidate" + ); + + // The mint: several brand-new repos. They sort last and are the only thing an + // attacker who can grind owner DIDs actually gets to do. + for i in 0..5 { + let name = format!("minted{i}"); + init_empty_bare( + &std::path::PathBuf::from("/tmp") + .join(&slug) + .join(format!("{name}.git")), + ); + state + .db + .create_repo(&seed_repo_at(&owner_did, &name, 100 + i)) + .await + .expect("seed a minted candidate"); + } + // And a whole window's worth entering BELOW the boundary, which is what an + // offset silently mistakes for a move of the boundary itself. + for i in 0..cap { + let name = format!("warmed{i}"); + init_empty_bare( + &std::path::PathBuf::from("/tmp") + .join(&slug) + .join(format!("{name}.git")), + ); + state + .db + .create_repo(&seed_repo_at(&owner_did, &name, -(i as i64) - 1)) + .await + .expect("seed a candidate below the boundary"); + } + + let second = tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the second traversal terminates"); + + assert_eq!( + second.repaired, 1, + "the window boundary is a key, so twenty-one candidates arriving between \ + traversals leave it exactly where the first traversal put it and the holder \ + is still the next thing read" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the holder's row is repaired despite the churn" + ); + } + + /// F5 scenario 3 (#173 round 13): a candidate list that has shrunk below one window + /// RESETS the continuation instead of stranding it past the end of the list. + /// + /// The migration's own success shrinks the list (repos go cold, get deleted), and a + /// continuation left pointing past everything would rotate each later traversal to an + /// empty tail and then wrap to the same prefix forever. Once the whole warm list fits + /// in one window there is no next window to advance to, so the traversal resets. + #[sqlx::test] + async fn sweep_discovery_shrunken_list_resets_the_continuation(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["shrinksrc"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("shrinksrc.git"); + // Nothing warm holds the object, so the row stays unrepaired and every traversal + // spends a full window on it. + let candidates = + seed_candidate_ladder(&state.db, &owner_did, &slug, "shrinkcand", cap + 1, None).await; + seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; + + tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the first traversal terminates"); + assert_eq!( + state.db.discovery_continuation().await.unwrap(), + candidate_key(&candidates[cap - 1]), + "precondition: the traversal parked the continuation past the head" + ); + + // The list shrinks to two: everything but the two oldest goes away. + for repo in candidates.iter().skip(2) { + sqlx::query("DELETE FROM repos WHERE id = $1") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + } + + tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the second traversal terminates"); + + assert_eq!( + state.db.discovery_continuation().await.unwrap(), + (String::new(), String::new()), + "once the whole warm list fits in one window there is no next window, so the \ + continuation resets rather than pointing past the end of a shrunken list" + ); + } + + /// F5 scenario 5 (#173 round 13, MUST-NOT): a traversal may only advance over + /// candidates it really READ, never over the ones it merely walked past with a dead + /// deadline. + /// + /// U3 gives each source-less row a slice of the pass budget and skips a row reached + /// with the pass budget already gone. What it does NOT skip is the candidates behind + /// a wedged one INSIDE a row: those still enter the probe loop, still get charged a + /// read, and still come back retryable, but `db_bounded` returns on the spent + /// deadline without touching the repo. Advancing over them would burn a window nobody + /// looked in, which is the same hole the continuation exists to close. + /// + /// Twenty warm candidates, seven source-less rows, and a `git` that wedges on + /// everything. With `git_timeout` at 4s each row slice is 1s, so each row that probes + /// at all spends its whole slice on candidate ONE and walks the other fifteen with a + /// dead deadline. The traversal may advance to candidate one and no further. + #[sqlx::test] + async fn sweep_discovery_starved_traversal_advances_only_over_live_probes(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["starvesrc"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("starvesrc.git"); + let candidates = + seed_candidate_ladder(&state.db, &owner_did, &slug, "starvecand", cap + 4, None).await; + for oid in [ + &fx.public_oid, + &fx.secret_oid, + &fx.public_tree_oid, + &fx.secret_tree_oid, + &fx.root_tree_oid, + &fx.commit_oid, + &fx.tag_oid, + ] { + seed_legacy_pin(&pool, &src, oid, None).await; + } + + let git_bin = write_git_shim( + &format!("gl-starve-git-{short}"), + "#!/bin/sh\nsleep 30\nexit 1\n", + ); + + tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + git_bin.to_str().unwrap(), + std::time::Duration::from_secs(4), + 16, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the traversal terminates") + .expect("the traversal succeeds"); + + let seen = state.db.discovery_continuation().await.unwrap(); + assert_eq!( + seen, + candidate_key(&candidates[0]), + "the only candidate any row read with a live deadline is the first, so that \ + is exactly how far the traversal may advance" + ); + assert_ne!( + seen, + candidate_key(&candidates[cap - 1]), + "advancing to the end of the window would skip fifteen candidates that were \ + charged a read but never actually looked at" + ); + } + + /// F5 scenario 6 (#173 round 13, MUST-NOT): a candidate that wedges MID-window must + /// not carry the continuation past the candidates behind it. + /// + /// The sharp version of the live-budget rule, and the one a window's-end advance gets + /// wrong while looking correct. Twenty-four warm candidates, the holder at position + /// twelve, and a `git` that wedges only in the repo at position nine. Positions one + /// to eight are read for real, nine eats the row's whole slice, and ten through + /// sixteen are charged a read apiece against a dead deadline without the repo ever + /// being opened. The continuation may advance to nine and no further, or the holder + /// at twelve is skipped by a traversal that never looked at it. + #[sqlx::test] + async fn sweep_discovery_hung_mid_window_does_not_skip_unread_candidates(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["midsrc", "midheld"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("midsrc.git"); + let candidates = seed_candidate_ladder( + &state.db, + &owner_did, + &slug, + "midcand", + cap + 8, + Some((12, "midheld")), + ) + .await; + let (raw_cid, provider_cid) = seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; + + // Wedges only inside the position-nine repo, which the sweep enters by cwd. + let git_bin = write_git_shim( + &format!("gl-mid-git-{short}"), + "#!/bin/sh\ncase \"$(pwd)\" in\n */midcand9.git) sleep 30; exit 1 ;;\nesac\nexec git \"$@\"\n", + ); + + let first = tokio::time::timeout( + std::time::Duration::from_secs(180), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + git_bin.to_str().unwrap(), + std::time::Duration::from_secs(16), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the first traversal terminates"); + + assert_eq!( + first.repaired, 0, + "the wedged candidate spends the row's slice, so the holder behind it is \ + charged a read but never actually read" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "precondition: the first traversal leaves the row on its provider key" + ); + assert_eq!( + state.db.discovery_continuation().await.unwrap(), + candidate_key(&candidates[8]), + "the last candidate read with a live deadline is the wedged one at position \ + nine, so that is the boundary; anything further skips unread candidates" + ); + + let second = tokio::time::timeout( + std::time::Duration::from_secs(180), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + git_bin.to_str().unwrap(), + std::time::Duration::from_secs(16), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the second traversal terminates"); + + assert_eq!( + second.repaired, 1, + "the next traversal starts at position ten and reaches the holder at twelve" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the row is repaired from the candidate the hang had hidden" + ); + } + + /// F5 scenario 7 (#173 round 13): the continuation survives the future being DROPPED. + /// + /// `spawn_legacy_cid_sweep` runs the sweep inside a `tokio::select!` against the + /// shutdown watcher, so on shutdown the sweep future is dropped wherever it happens + /// to be. The re-arm wrapper never returns on a healthy node, so a persist written on + /// the way out of the wrapper would never be written at all. Persisting inside the + /// traversal-ending pass is what makes a shutdown cost at most a repeated window. + #[sqlx::test] + async fn sweep_continuation_survives_dropped_sweep_future(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["dropsrc"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dropsrc.git"); + let candidates = + seed_candidate_ladder(&state.db, &owner_did, &slug, "dropcand", cap + 1, None).await; + seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; + + // The wrapper completes a traversal and then parks on its re-arm sleep, which is + // exactly where a shutdown drops it in production. + let expected = candidate_key(&candidates[cap - 1]); + let observed = tokio::select! { + _ = crate::ipfs_pin::run_sweep_rearmed( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + std::time::Duration::from_secs(3600), + &state.db, + ) => None, + v = poll_until(std::time::Duration::from_secs(120), || async { + let c = state.db.discovery_continuation().await.unwrap(); + (c != (String::new(), String::new())).then_some(c) + }) => v, + }; + + assert_eq!( + observed, + Some(expected), + "the traversal-ending pass persists the continuation, so dropping the sweep \ + future afterwards keeps the advance the traversal earned" + ); + } + + /// F5 scenario 8 (#173 round 13, MUST-NOT, the aliasing negative): two source-less + /// rows in DIFFERENT passes of the same traversal share one window. + /// + /// The continuation advances once per TRAVERSAL, not once per pass. Per pass, a row's + /// window index across traversals is `(t*m + j) mod W` for `m` rows and `W` windows, + /// so with `m` and `W` sharing a factor a given row only ever visits `W / gcd(m, W)` + /// of the windows and orbits a strict subset forever. Two rows, two windows: under + /// per-pass advancement the first row is pinned to window one for the life of the + /// node and its holder in window two is unreachable. + /// + /// Thirty-two warm candidates, `batch = 1` so the two rows land in different passes, + /// the first row's holder at position twenty, the second row's object nowhere at all. + #[sqlx::test] + async fn sweep_discovery_rows_in_different_batches_share_windows(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["alisrc"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("alisrc.git"); + // The walk is ordered by `sha256_hex`, so the smaller oid is the row read first. + let mut oids = [fx.public_oid.clone(), fx.secret_oid.clone()]; + oids.sort(); + let (first_row, second_row) = (oids[0].clone(), oids[1].clone()); + + let candidates = + seed_candidate_ladder(&state.db, &owner_did, &slug, "alicand", 2 * cap, None).await; + // Position twenty holds ONLY the first row's blob: a bare clone would carry both + // and the second row would stop being the unrepairable control. + copy_blob_into_bare( + &src, + &std::path::PathBuf::from("/tmp") + .join(&slug) + .join("alicand20.git"), + &first_row, + ); + let (first_raw, first_provider) = seed_legacy_pin(&pool, &src, &first_row, None).await; + seed_legacy_pin(&pool, &src, &second_row, None).await; + + let log = std::env::temp_dir().join(format!("gl-ali-log-{short}")); + let _ = std::fs::remove_file(&log); + let git_bin = write_git_shim( + &format!("gl-ali-git-{short}"), + &format!( + "#!/bin/sh\n\ + if [ \"$2\" = \"--batch-check\" ]; then\n\ + \x20 oid=$(cat)\n\ + \x20 printf '%s %s\\n' \"$oid\" \"$(basename $(pwd))\" >> {log}\n\ + \x20 printf '%s\\n' \"$oid\" | git \"$@\"\n\ + \x20 exit $?\n\ + fi\n\ + exec git \"$@\"\n", + log = log.display() + ), + ); + + let mut traversal = crate::ipfs_pin::DiscoveryTraversalState::default(); + for _ in 0..2 { + tokio::time::timeout( + std::time::Duration::from_secs(180), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + git_bin.to_str().unwrap(), + git_timeout, + 1, + std::time::Duration::ZERO, + &state.db, + &mut traversal, + ), + ) + .await + .expect("the traversal terminates"); + } + + assert_eq!( + stored_pin(&pool, &first_row).await.0, + first_raw, + "the first row's holder is in the second window, which it only reaches if \ + both rows moved through the windows together" + ); + assert_ne!( + first_raw, first_provider, + "control: the repaired key really differs from the seeded legacy one" + ); + + // The invocation log proves the shared window rather than inferring it: inside + // one traversal both oids must have been probed against the same repos. + let text = std::fs::read_to_string(&log).expect("the shim logged its probes"); + let _ = std::fs::remove_file(&log); + let repos_for = |oid: &str| -> Vec { + let mut v: Vec = text + .lines() + .filter_map(|l| l.split_once(' ')) + .filter(|(o, _)| *o == oid) + .map(|(_, r)| r.to_string()) + .collect(); + v.sort(); + v.dedup(); + v + }; + let first_probed = repos_for(&first_row); + let second_probed = repos_for(&second_row); + assert!( + first_probed.len() >= cap && second_probed.len() >= cap, + "precondition: both rows really probed a full window ({} and {})", + first_probed.len(), + second_probed.len() + ); + let window_one: Vec = { + let mut v: Vec = candidates[..cap] + .iter() + .map(|r| format!("{}.git", r.name)) + .collect(); + v.sort(); + v + }; + for repo in &window_one { + assert!( + first_probed.contains(repo) && second_probed.contains(repo), + "both rows must have probed {repo} in the first traversal; per-pass \ + advancement would have handed the second row a different window" + ); + } + } + + /// F5 scenario 9 (#173 round 13): the dead-read cap PAUSES a run, and the re-arm is + /// what keeps coverage moving afterwards. + /// + /// Before the wrapper the sweep ran exactly once per boot, so the window advanced + /// once per boot too and a node that never reboots never advanced past its first + /// window. Five unrepairable source-less rows at a full window of probes each blow + /// through `MAX_DEAD_ROW_READS_PER_RUN` inside one run; the holder sits in the second + /// window, so it is reachable only across re-arms. + #[sqlx::test] + async fn sweep_rearm_advances_past_dead_read_cap(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["rearmsrc", "rearmheld"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("rearmsrc.git"); + seed_candidate_ladder( + &state.db, + &owner_did, + &slug, + "rearmcand", + cap + 4, + Some((cap + 2, "rearmheld")), + ) + .await; + let mut oids = [ + fx.public_oid.clone(), + fx.secret_oid.clone(), + fx.public_tree_oid.clone(), + fx.secret_tree_oid.clone(), + fx.root_tree_oid.clone(), + ]; + oids.sort(); + for oid in &oids { + seed_legacy_pin(&pool, &src, oid, None).await; + } + let target = oids.last().unwrap().clone(); + let target_raw = { + let (_ty, bytes) = crate::git::store::read_object(&src, &target) + .expect("read the object") + .expect("the object exists"); + gitlawb_core::cid::Cid::from_git_object_bytes(&bytes).to_string() + }; + + // One run, driven directly: five rows at a full window each is more fruitless + // reading than one run will do, so it PAUSES rather than completing. + let mut traversal = crate::ipfs_pin::DiscoveryTraversalState::default(); + let run = tokio::time::timeout( + std::time::Duration::from_secs(300), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 1, + std::time::Duration::ZERO, + &state.db, + &mut traversal, + ), + ) + .await + .expect("the run terminates"); + assert_eq!( + run.stop, + crate::ipfs_pin::SweepStop::PausedOnDeadReadCap, + "one run cannot walk this table: it stops on the dead-read cap with the \ + cursor mid-table" + ); + assert_eq!(run.repaired, 0, "the holder is past the first window"); + + let repaired = tokio::select! { + _ = crate::ipfs_pin::run_sweep_rearmed( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 1, + std::time::Duration::ZERO, + std::time::Duration::ZERO, + &state.db, + ) => false, + v = poll_until(std::time::Duration::from_secs(300), || async { + (stored_pin(&pool, &target).await.0 == target_raw).then_some(()) + }) => v.is_some(), + }; + assert!( + repaired, + "a run that pauses on the dead-read cap is re-armed, so traversals keep \ + completing and the window keeps advancing until the holder is reached" + ); + } + + /// F5 scenario 10 (#173 round 13, MUST-NOT): a cap pause AFTER the traversal's last + /// source-less row must not lose the advance that traversal earned. + /// + /// The traversal accumulator is scoped to the TRAVERSAL, not the run, and this is the + /// case that separates the two. The run that probes the window is paused by the + /// dead-read cap before it reaches the end of the table; a LATER run reads the short + /// batch and is the one that applies the advance. Rebuild the accumulator per run and + /// that later run sees nothing recorded, applies the hold arm, and the window never + /// moves however many times the sweep re-arms. + /// + /// One source-less row against twenty warm candidates (a full window of probes), + /// then enough bytes-gone rows behind it to trip the cap before the short batch. + #[sqlx::test] + async fn sweep_cap_pause_after_last_discovery_row_still_advances(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let cap = crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES; + + let fx = seed_cid_repos(&slug, &short, &["pausesrc"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pausesrc.git"); + let candidates = + seed_candidate_ladder(&state.db, &owner_did, &slug, "pausecand", cap + 4, None).await; + let (_raw, provider) = seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; + assert!( + !fx.public_oid.starts_with("ff"), + "precondition: the discovery row sorts before the synthetic bytes-gone rows" + ); + + // Bytes-gone rows: real provenance pointing at a warm repo that does not hold + // them, so each costs exactly one fruitless read. Enough of them that the run + // trips the cap with the end of the table still ahead of it. + let ghost = &candidates[0]; + let needed = crate::ipfs_pin::MAX_DEAD_ROW_READS_PER_RUN - cap; + for i in 0..needed { + let oid = format!("ff{:062x}", i); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) + VALUES ($1, $2, $3, NULL)", + ) + .bind(&oid) + .bind(&provider) + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + state + .db + .record_pin_source(&oid, &ghost.id) + .await + .expect("record the ghost source"); + } + + let expected = candidate_key(&candidates[cap - 1]); + let observed = tokio::select! { + _ = crate::ipfs_pin::run_sweep_rearmed( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 1, + std::time::Duration::ZERO, + std::time::Duration::ZERO, + &state.db, + ) => None, + v = poll_until(std::time::Duration::from_secs(300), || async { + let c = state.db.discovery_continuation().await.unwrap(); + (c != (String::new(), String::new())).then_some(c) + }) => v, + }; + + assert_eq!( + observed, + Some(expected), + "the run that probed the window was paused by the dead-read cap, so a LATER \ + run ends the traversal; the advance it applies has to come from the \ + traversal's accumulator, not that run's" + ); + } + + /// F5 scenario 11 (#173 round 13): a failing pass query EXITS the wrapper. + /// + /// The re-arm loop is for a healthy node making slow progress. A broken database is + /// not that, and retrying it every re-arm would turn one logged failure into an + /// endless stream of them. The one-shot behavior a failing pass had before the + /// wrapper existed is preserved exactly. + #[sqlx::test] + async fn sweep_rearm_exits_on_pass_failure(pool: PgPool) { + let state = test_state(pool.clone()).await; + sqlx::query("DROP TABLE pinned_cids") + .execute(&pool) + .await + .unwrap(); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::run_sweep_rearmed( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(5), + 16, + std::time::Duration::ZERO, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("a failing pass query must END the wrapper, never re-arm it forever"); + + assert_eq!( + stats.stop, + crate::ipfs_pin::SweepStop::PassFailed, + "the wrapper reports the failure it exited on" + ); + assert_eq!( + stats.repaired, 0, + "nothing was repaired against a broken table" + ); + } + /// F1 scenario 6 (#173, the collision case): two warm repos hold identical bytes, /// which is the shape (forks, a shared LICENSE blob, the empty tree) that makes an /// exclusive first-pinner claim wrong. Discovery records ONE additive source and @@ -8365,6 +9406,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -8438,6 +9480,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -8494,6 +9537,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -8703,6 +9747,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ) .await; assert_eq!( @@ -8845,6 +9890,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await @@ -9074,6 +10120,7 @@ mod tests { blocker.commit().await.expect("release the row lock"); parked }; + let mut traversal = Default::default(); let (stats, parked) = tokio::time::timeout(std::time::Duration::from_secs(60), async { tokio::join!( crate::ipfs_pin::sweep_legacy_provider_cids( @@ -9083,6 +10130,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut traversal, ), driver ) @@ -9155,6 +10203,7 @@ mod tests { state.db.create_repo(&holder).await.expect("seed holder"); let (raw_cid, _provider) = seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + let (mut ta, mut tb) = (Default::default(), Default::default()); let (a, b) = tokio::time::timeout(std::time::Duration::from_secs(60), async { tokio::join!( crate::ipfs_pin::sweep_legacy_provider_cids_once( @@ -9163,6 +10212,7 @@ mod tests { git_timeout, 16, &state.db, + &mut ta, ), crate::ipfs_pin::sweep_legacy_provider_cids_once( std::path::Path::new("/tmp"), @@ -9170,6 +10220,7 @@ mod tests { git_timeout, 16, &state.db, + &mut tb, ) ) }) @@ -9250,6 +10301,7 @@ mod tests { 16, std::time::Duration::ZERO, &state.db, + &mut Default::default(), ), ) .await From 9babb5ece053608c522f49b4528de23d5b61b749 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:13:17 -0500 Subject: [PATCH 50/77] fix(node): record a discovered source and its marker in one commit Discovery recorded the source and armed the fallback in two independent best-effort writes. If the insert landed and the marker did not, the row was left raw CIDv1 with a nonempty, below-cap, unmarked source set. Later sweeps skip such a row, and get_by_cid reads that set as complete and suppresses the fallback scan, so once the transient error cleared, a public duplicate of an object whose recorded holder is private stayed unreachable for good. record_discovered_pin_source does both writes in one transaction: the capped source insert, the gated failure-row delete, and the sentinel that arms the fallback. The sentinel is unconditional because one holder found by a bounded warm-only probe never proves the set complete. The method ends in an explicit commit, so an elapsed deadline is a definite non-write rather than an unknown. A failed record still leaves an empty source set, which the resolver's own empty-set signal routes to the fallback, so the degradation posture is unchanged and now has a test. The fault injection is a BEFORE INSERT trigger on pin_source_failures, not a dropped table. Dropping it would fail the delete inside record_pin_source's own transaction, roll that back, and produce the same empty set as the fix, so the test would pass without ever reaching the half-state. Verified by mutation: splitting the transaction back into two writes 404s the public copy, and dropping the sentinel leaves the fallback unarmed. --- crates/gitlawb-node/src/db/mod.rs | 64 ++++++++ crates/gitlawb-node/src/ipfs_pin.rs | 115 +++++++++++++-- crates/gitlawb-node/src/test_support.rs | 187 ++++++++++++++++++++++++ 3 files changed, 350 insertions(+), 16 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index d2b6206d..1eb08e6b 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -3118,6 +3118,70 @@ impl Db { Ok(()) } + /// Record a DISCOVERED holder and arm the resolver's fallback ATOMICALLY (U5, #173). + /// The sweep's discovery arm used to call `record_pin_source` and then, separately, + /// `mark_pin_sources_incomplete`. Two best-effort writes, so a transient failure of + /// the second one left the row with a nonempty, below-cap, UNMARKED source set: the + /// resolver's `needs_scan` is `sources.is_empty() || at_cap || incomplete`, so all + /// three signals were off, the bounded legacy scan was dropped, and an unrecorded + /// public duplicate stayed 404'd for good once the DB error cleared (no later sweep + /// revisits a raw-CIDv1 row). One transaction removes that state entirely: either the + /// source row and the sentinel both land or neither does, and neither-lands is the + /// benign end (an empty set is itself a `needs_scan` signal). + /// + /// The sentinel insert is UNCONDITIONAL, unlike the marker clear's `rows_affected` + /// gate: discovery probes a bounded, warm-only candidate set and stops at the first + /// holder, so finding one holder is never evidence the set is complete, whether or + /// not this particular call added a row. It names the empty-string UNKNOWN-repo + /// sentinel (the same one the v24 migration carries pre-upgrade markers under), so no + /// real per-repo record can clear it, and it carries the same + /// `WHERE EXISTS (pinned_cids row)` guard as [`Self::mark_pin_sources_incomplete`] + /// so a marker never sits in the table for an object this node never pinned. + /// + /// Commit-terminated, like [`Self::record_pin_source`], so a caller that wraps this + /// in `db_bounded` may read `BoundedDbError::Elapsed` as "definitely did not land": + /// the cancelled future never reaches `tx.commit()`, no COMMIT is sent, and Postgres + /// discards the transaction when the connection resets. + pub async fn record_discovered_pin_source( + &self, + sha256_hex: &str, + repo_id: &str, + ) -> Result<()> { + let mut tx = self.pool.begin().await?; + let inserted = sqlx::query( + "INSERT INTO pin_repo_sources (sha256_hex, repo_id) + SELECT $1, $2 + WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo_id) + .bind(MAX_PIN_SOURCES) + .execute(&mut *tx) + .await? + .rows_affected(); + if inserted > 0 { + // Clears THIS repo's failure only, the same gate and reason as + // `record_pin_source`: a per-object clear let one repo's genuine record wipe + // a marker another repo's failure set. + sqlx::query("DELETE FROM pin_source_failures WHERE sha256_hex = $1 AND repo_id = $2") + .bind(sha256_hex) + .bind(repo_id) + .execute(&mut *tx) + .await?; + } + sqlx::query( + "INSERT INTO pin_source_failures (sha256_hex, repo_id) + SELECT $1, '' WHERE EXISTS (SELECT 1 FROM pinned_cids WHERE sha256_hex = $1) + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + /// Mark this object's pin-source set as KNOWN INCOMPLETE for `repo_id` (U3, #173). /// Called when a `record_pin_source` exhausts its retries, which is the only moment /// the node knows a source it meant to record is missing. `GET /ipfs/{cid}` reads it diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 2fb53cfe..628d828c 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -610,12 +610,18 @@ enum DiscoveryOutcome { /// incomplete marker goes with it because one discovered holder never proves the set /// complete. /// -/// Both writes are best-effort and warn-only, and the degradation is stated rather -/// than deferred to a healing pass that does not exist: if the source record fails the -/// row is raw-CIDv1 with an empty or incomplete source set, which is exactly the state -/// `needs_scan` routes to the bounded legacy scan, so the object stays servable. The -/// sweep itself never revisits it (the cost gate skips a raw row free from then on), -/// so the resolver's fallback is the healing path, not a retry. +/// Both rows are written by ONE transaction (`record_discovered_pin_source`, U5), never +/// as two independent best-effort calls. Split, a failed sentinel left the row with a +/// nonempty, below-cap, UNMARKED source set, which `needs_scan` reads as complete: the +/// fallback scan is dropped and an unrecorded public duplicate is 404'd permanently. +/// Together they either both land or neither does. +/// +/// The record as a whole is still best-effort and warn-only, and the degradation is +/// stated rather than deferred to a healing pass that does not exist: if it fails the row +/// is raw-CIDv1 with an EMPTY source set, which is exactly the state `needs_scan` routes +/// to the bounded legacy scan, so the object stays servable. The sweep itself never +/// revisits it (the cost gate skips a raw row free from then on), so the resolver's +/// fallback is the healing path, not a retry. async fn discover_legacy_row( sha: &str, ctx: &mut Option>, @@ -684,12 +690,11 @@ async fn discover_legacy_row( reads += 1; match repair_legacy_provider_cid(repo_path, git_bin, row_deadline, sha, db).await { Ok(RepairOutcome::Repaired) => { - if let Err(e) = db.record_pin_source(sha, &repo.id).await { - tracing::warn!(sha = %sha, repo_id = %repo.id, err = %e, "sweep discovery: failed to record the discovered pin source"); - } - // Discovery found ONE holder out of a bounded, warm-only candidate set, - // so the source set is still not known complete and the resolver must - // keep its scan fallback for this row. + // ONE transaction for both writes (U5, #173). Discovery found ONE holder + // out of a bounded, warm-only candidate set, so the source set is still + // not known complete and the resolver must keep its scan fallback for + // this row; the sentinel that arms it is therefore not a separate + // best-effort write but part of the same commit as the source row. // // Marked against the UNKNOWN-repo sentinel rather than the repo just // recorded, which would be a lie (that repo IS recorded). The sentinel is @@ -699,10 +704,35 @@ async fn discover_legacy_row( // // Rebase note (#321 onto the per-(oid, repo) marker): the original wrote // this marker because `record_pin_source` used to clear the whole - // per-object boolean. It no longer does, so this call went from - // compensating for a clear to being the only thing arming the fallback. - if let Err(e) = db.mark_pin_sources_incomplete(sha, "").await { - tracing::warn!(sha = %sha, err = %e, "sweep discovery: failed to mark the pin-source set incomplete"); + // per-object boolean. It no longer does, so the sentinel went from + // compensating for a clear to being the only thing arming the fallback, + // which is why it may not be allowed to fail on its own. + match db_bounded( + db_record_deadline(row_deadline), + retry_db_record(|| db.record_discovered_pin_source(sha, &repo.id)), + ) + .await + { + Ok(()) => {} + // Elapsed is a DEFINITE non-write here, and that follows from the + // shape of what was wrapped: the record is commit-terminated, so a + // cancelled future never sends the COMMIT. The arm stays separate + // only so the warn tells an operator a stalled DB from a scattered + // per-row failure; both leave the same benign end state below. + Err(e @ BoundedDbError::Elapsed) => { + tracing::warn!( + sha = %sha, + repo_id = %repo.id, + err = %e, + "sweep discovery: the discovered pin source record did not \ + complete inside the row deadline; a cancelled \ + commit-terminated transaction definitely did not land, so \ + the row keeps an empty source set and the resolver falls back" + ); + } + Err(e) => { + tracing::warn!(sha = %sha, repo_id = %repo.id, err = %e, "sweep discovery: failed to record the discovered pin source and its sentinel"); + } } traversal.note_row(live_probes, fits_under_cap); return (DiscoveryOutcome::Repaired, reads); @@ -2716,6 +2746,59 @@ mod tests { ); } + /// U5 (#173): the elapsed arm of the discovery record leaves neither row behind. + /// + /// What this proves and what it does NOT: it shows the wrapper composition + /// (`db_bounded` over `retry_db_record` over `record_discovered_pin_source`) returns + /// promptly and cleanly on the elapsed arm with a healthy pool, so the call site's + /// "definitely did not land" reading is not contradicted here. It is NOT evidence of + /// transactionality: a spent deadline reduces to `timeout(0, fut)`, the future never + /// starts, and "neither row landed" would hold just as well for two separate calls. + /// It kills no mutation. The atomicity property is proven by + /// `sweep_discovery_failed_marker_does_not_strand_public_copy` and its mutations + /// alone. + /// + /// Driven directly with a past deadline rather than through `db_record_deadline`, + /// whose `DB_RECORD_GRACE` floor makes this arm near-unreachable in production. + #[sqlx::test] + async fn discovery_record_elapsed_leaves_neither_row(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool.clone()); + db.run_migrations().await.expect("migrations"); + let sha = "d5".repeat(32); + // A `pinned_cids` row so the sentinel's `WHERE EXISTS` guard is satisfied and its + // absence below is the timeout's doing, not the guard's. + db.record_pinned_cid_with_source(&sha, &seed_cid(), "repo-first") + .await + .expect("seed the pinned row"); + + let spent = Instant::now() - Duration::from_secs(5); + let started = std::time::Instant::now(); + let out = db_bounded( + spent, + retry_db_record(|| db.record_discovered_pin_source(&sha, "repo-discovered")), + ) + .await; + + assert!( + matches!(out, Err(BoundedDbError::Elapsed)), + "a spent deadline must yield the timeout arm, not a value: {out:?}" + ); + assert!( + started.elapsed() < Duration::from_secs(1), + "the wrapped retry ladder must not outlive the spent deadline; got {:?}", + started.elapsed() + ); + assert_eq!( + db.pin_sources_for_oid(&sha).await.unwrap(), + vec!["repo-first".to_string()], + "no discovered source row landed" + ); + assert!( + !db.pin_sources_incomplete(&sha).await.unwrap(), + "no sentinel landed either" + ); + } + /// The ABSOLUTE half of the same helper, which no loop-level test actually binds. /// /// `db_bounded` takes an `Instant`, not a `Duration`, so every call sharing one diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index c2bb0884..58a8da43 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -9449,6 +9449,193 @@ mod tests { assert!(body.contains("public bytes"), "the object's bytes serve"); } + /// U5 (#173, F7): the discovered-source row and the fallback-arming sentinel commit + /// together or not at all, so discovery can never leave a row in the one state the + /// resolver reads as complete while it is not: a nonempty, below-cap, UNMARKED + /// source set. + /// + /// Same shape as the multi-holder test above (older private holder selected, newer + /// public holder unrecorded), plus a fault that fails ONLY the marker insert: a + /// `BEFORE INSERT` trigger on `pin_source_failures` that raises. Under the pre-fix + /// two-call shape the source insert commits (its own transaction's second statement + /// is a DELETE, which an insert trigger does not fire) and the separate marker insert + /// then fails, leaving the source set holding the private holder alone with no + /// marker; `needs_scan` is `sources.is_empty() || at_cap || incomplete`, so all three + /// signals are off, the resolver drops its fallback scan, and the public duplicate is + /// permanently 404'd for the anonymous caller. One transaction makes that state + /// unreachable: the marker's failure rolls the source row back with it, the set stays + /// EMPTY, and the empty-set signal routes the request to the fallback scan. + #[sqlx::test] + async fn sweep_discovery_failed_marker_does_not_strand_public_copy(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["stranda", "strandb"]); + let bare_a = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("stranda.git"); + + // A is older, so the oldest-first probe order selects it; it is PRIVATE, so it + // denies the anonymous caller. B is public and holds the same bytes. + let mut repo_a = seed_repo(&owner_did, "stranda"); + repo_a.is_public = false; + repo_a.created_at = Utc::now() - chrono::Duration::days(2); + let repo_b = seed_repo(&owner_did, "strandb"); + state.db.create_repo(&repo_a).await.expect("seed repo a"); + state.db.create_repo(&repo_b).await.expect("seed repo b"); + + let (raw_cid, _provider) = seed_legacy_pin(&pool, &bare_a, &fx.public_oid, None).await; + + // The fault, installed AFTER migrations: every insert into `pin_source_failures` + // raises. Postgres triggers cannot raise inline, hence the plpgsql function. + // Deliberately NOT a `DROP TABLE`: the source-record transaction's own DELETE on + // this table would then error too, that transaction would roll back, and the + // pre-fix run would land the same empty set as the post-fix one, so the test + // would pass for the wrong reason. + sqlx::query( + "CREATE FUNCTION fail_pin_source_failure_insert() RETURNS trigger AS $$ + BEGIN RAISE EXCEPTION 'injected pin_source_failures insert failure'; END; + $$ LANGUAGE plpgsql", + ) + .execute(&pool) + .await + .expect("install the fault function"); + sqlx::query( + "CREATE TRIGGER fail_pin_source_failure_insert + BEFORE INSERT ON pin_source_failures + FOR EACH ROW EXECUTE FUNCTION fail_pin_source_failure_insert()", + ) + .execute(&pool) + .await + .expect("install the fault trigger"); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates"); + assert_eq!( + stats.repaired, 1, + "the row is still repaired to its raw key; only the source record is at risk" + ); + + // Gathered before the assertions so the RED output carries the half-state. + let sources = state + .db + .pin_sources_for_oid(&fx.public_oid) + .await + .expect("read the source set"); + let marker_rows: i64 = + sqlx::query_scalar("SELECT count(*) FROM pin_source_failures WHERE sha256_hex = $1") + .bind(&fx.public_oid) + .fetch_one(&pool) + .await + .expect("read the marker table"); + + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + (st, body.contains("public bytes")), + (StatusCode::OK, true), + "the anonymous caller must be served the PUBLIC holder's copy through the \ + fallback scan; got {st} with source set {sources:?} and {marker_rows} marker \ + row(s), the nonempty-and-unmarked half-state the resolver reads as complete" + ); + assert!( + sources.is_empty(), + "the failed marker must roll the source row back with it; got {sources:?}" + ); + assert_eq!( + marker_rows, 0, + "the marker insert is what failed, so no marker row can exist" + ); + } + + /// U5 (#173, F7, the healthy direction): one discovery hit writes BOTH rows, asserted + /// against the tables directly rather than through the boolean helper. The sentinel + /// is unconditional because one discovered holder out of a bounded warm-only + /// candidate set never proves the source set complete, and it is written against the + /// empty-string UNKNOWN-repo sentinel so no later real record clears it. + #[sqlx::test] + async fn sweep_discovery_records_source_and_sentinel_in_one_commit(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["bothrows"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("bothrows.git"); + let repo = seed_repo(&owner_did, "bothrows"); + state.db.create_repo(&repo).await.expect("seed repo"); + + seed_legacy_pin(&pool, &bare, &fx.public_oid, None).await; + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the sweep terminates"); + assert_eq!(stats.repaired, 1, "the discovered row is repaired"); + + let source_rows: Vec = sqlx::query_scalar( + "SELECT repo_id FROM pin_repo_sources WHERE sha256_hex = $1 ORDER BY repo_id", + ) + .bind(&fx.public_oid) + .fetch_all(&pool) + .await + .expect("read pin_repo_sources"); + assert_eq!( + source_rows, + vec![repo.id.clone()], + "the discovered holder is recorded additively" + ); + + let marker_repos: Vec = sqlx::query_scalar( + "SELECT repo_id FROM pin_source_failures WHERE sha256_hex = $1 ORDER BY repo_id", + ) + .bind(&fx.public_oid) + .fetch_all(&pool) + .await + .expect("read pin_source_failures"); + assert_eq!( + marker_repos, + vec![String::new()], + "the same commit writes the unknown-repo sentinel: one discovered holder never \ + proves the set complete, so the resolver must keep its fallback scan" + ); + } + /// F1 scenario 7 (#173, degenerate state): the cost gate at the top of the row loop /// fires before the sources query, so a source-less row that is ALREADY raw-CIDv1 /// never enters discovery and reads nothing. From 22f4db94b2fb3b3c7c43c5bd34608ea77ce5d197 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:49:51 -0500 Subject: [PATCH 51/77] fix(node): bound the legacy CID scan and let the caller carry its position A denial-only inventory paged through the whole repo table. gate_and_serve returns Skip on quarantine and on the root visibility deny before either walk.probes or walk.visits increments, and those two counters were the pager's only stop conditions, so an anonymous caller holding a CID from the public pins index could read every row and its rules while the global IPFS walk admission stayed held for the request budget. Three parts, and the shape of each is set by a way an earlier attempt was wrong. A row ceiling, with a rules companion. Rows scanned bounds the DB reads and the pager's row retention; it does not bound rules, since the rules query has no LIMIT and an owner picks how many rules their repos carry, so that gets its own ceiling. Both taint into the existing retryable 503 and never a 404. A ceiling alone would stop there and strand content. The scan starts from a NULL cursor over (created_at, id) ASC, so a bare ceiling cuts the same oldest prefix on every request and a holder past it is unservable for good, which is the defect this PR already carries a finding about. So the 503 hands back the position. Server-side alternatives were tried and dropped: a node-global cursor lets concurrent callers skip each other's windows, and a per-caller map lets a key farm evict an honest caller mid-ladder. State the caller carries has neither problem. The token is sealed, not encoded. It names a row the caller may not be allowed to see, and id derives from the owner DID, so plaintext would leak a withheld repo's existence, creation time and owner. XChaCha20-Poly1305 with a fresh nonce per seal, the canonical CID as associated data, an expiry inside the sealed bytes, and a fixed-width plaintext so token length cannot vary with the row it names. Undecryptable, expired, malformed and foreign-CID tokens are one indistinguishable case: absent, start at the front. The wrap taint keys on the pager reaching exhaustion while resumed, not on any break site, so a token pointing at the last row still sheds a retryable 503 instead of falling through to the 404 tail. Pages are charged to the per-IP work bucket. Without that the ceiling bounds one request and nothing across requests, and the denial path pays nothing, which is the per-request-free-allowance shape already rejected once on this branch. The derived work-budget floor grows by the same page term so a deep scan cannot self-throttle mid-ladder. A toll shed renders 429 rather than 503, which reverses the tail's documented precedence for this one case. The tail comment names the exception: the caller's own bucket stopped the scan, not the node's search, and a 503 would invite an immediate retry into an empty bucket. Four existing tests hand-size the work limiter to their exact probe count and now also pay for a page; each gained one token and a comment showing the arithmetic. Production is unaffected because the floor grew by the same term. Verified by mutation: removing the ceiling unbounds the scan; truncating without tainting turns it into a 404; emitting a token but never resuming never reaches the holder; dropping the toll makes pages free; encoding the token as base64 plaintext leaks the withheld fields; dropping the CID from the associated data honors a cross-CID resume; a variable-width encoding makes token length vary with the row; and a constant nonce makes two seals identical. --- .env.example | 10 + README.md | 1 + crates/gitlawb-core/src/lib.rs | 1 + crates/gitlawb-core/src/scan_token.rs | 269 +++++ crates/gitlawb-node/src/api/ipfs.rs | 1407 ++++++++++++++++++++++- crates/gitlawb-node/src/auth/mod.rs | 3 + crates/gitlawb-node/src/config.rs | 68 +- crates/gitlawb-node/src/error.rs | 35 +- crates/gitlawb-node/src/main.rs | 5 + crates/gitlawb-node/src/state.rs | 56 +- crates/gitlawb-node/src/test_support.rs | 46 +- 11 files changed, 1859 insertions(+), 42 deletions(-) create mode 100644 crates/gitlawb-core/src/scan_token.rs diff --git a/.env.example b/.env.example index 174319a3..3126c8c7 100644 --- a/.env.example +++ b/.env.example @@ -217,6 +217,16 @@ GITLAWB_IPFS_WALK_PER_SOURCE=4 # scan-fallback fan-out (git cat-file per candidate repo) for an anonymous caller. A # truncated scan sheds a retryable 503, never a false 404. Default 256. GITLAWB_IPFS_MAX_LEGACY_PROBES=256 +# Max repo ROWS one /ipfs request's legacy scan may read from the database. The probe +# ceiling above only counts once a probe runs, and quarantined or private repos are +# denied before that, so without this an all-denying inventory paged the whole repo +# table for one anonymous request. A truncated scan sheds a retryable 503 carrying a +# sealed `continuation` token; echo it as ?scan= to resume, so a holder buried past the +# ceiling is reachable in ceil(repos / ceiling) + 1 requests. Raising this also raises +# the per-caller /ipfs work allowance, since each page is charged to it. Lowering it +# sharpens a coarse oracle: laddering to the end reveals the node's total repo count to +# within one ceiling. Default 2048. +GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS=2048 # Max EXPENSIVE path-scope visibility walks per single /ipfs request (only a # blob in a path-scoped repo costs a full-history walk). Over-cap repos are # skipped without a verdict and the scan continues; if the object is then found diff --git a/README.md b/README.md index 3b2e1047..cb988551 100644 --- a/README.md +++ b/README.md @@ -353,6 +353,7 @@ Important node settings: | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | | `GITLAWB_IPFS_MAX_LEGACY_PROBES` | Max legacy (NULL-provenance) repos probed per `/ipfs/{cid}` request, bounding the scan-fallback fan-out. A truncated scan returns a retryable 503, not a false 404. Default 256. | +| `GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS` | Max repo rows one `/ipfs/{cid}` request's legacy scan may read from the database. The probe ceiling above only starts counting once a probe runs, and quarantined or private repos are denied before that, so this is what bounds a scan over an inventory that denies the caller everywhere. A truncated scan sheds a retryable 503 carrying an opaque `continuation` token; echoing it as `?scan=` resumes the scan, so a holder buried past the ceiling is served within `ceil(repos / ceiling) + 1` requests and no ceiling ever produces a 404. Every page is charged to the caller's `/ipfs` work allowance, so raising this raises that allowance too. Lowering it sharpens an oracle: because a truncation emits a token and a completed wrap does not, laddering to the end reveals the node's total repo count (private and quarantined included) to within one ceiling. Default 2048. | | `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. The effective cap is the tighter of this knob and the node's internal per-request history-walk ceiling of 17 (`MAX_PIN_SOURCES + 1`), so a value above 17 has no effect while a value below it does tighten the cap. Default 64. | | `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate. Also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | | `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting and each also run under their own deadline (the lesser of `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` and the remaining budget), reaped via process-group teardown, so a hung `cat-file` cannot hold the request's walk slot past it. One hang path is still unbounded: the probe's object-store readability check is a plain filesystem sweep with nothing to reap, so a wedged filesystem can hold the slot past the deadline. Default 600. Accepted range is 1 to 3153600000 (100 years), since the node derives a deadline from this value and a larger one cannot be represented. | diff --git a/crates/gitlawb-core/src/lib.rs b/crates/gitlawb-core/src/lib.rs index efa99897..bda927be 100644 --- a/crates/gitlawb-core/src/lib.rs +++ b/crates/gitlawb-core/src/lib.rs @@ -6,6 +6,7 @@ pub mod error; pub mod http_sig; pub mod identity; pub mod sanitize; +pub mod scan_token; pub mod ucan; pub use error::Error; diff --git a/crates/gitlawb-core/src/scan_token.rs b/crates/gitlawb-core/src/scan_token.rs new file mode 100644 index 00000000..3ec55cd9 --- /dev/null +++ b/crates/gitlawb-core/src/scan_token.rs @@ -0,0 +1,269 @@ +//! Sealed continuation token for the node's bounded legacy CID scan (INV-13). +//! +//! The `/ipfs/{cid}` resolver's legacy scan stops at a row ceiling and sheds a +//! retryable 503. To let a holder buried past that ceiling still be reached, the +//! shed carries the scan position so the caller can echo it back and resume. The +//! whole point of the design is that the node keeps NO server-side scan state: the +//! position rides in the caller's token. +//! +//! That makes the token an EMITTED continuation derived from a FETCHED row, and on +//! a scan that served nothing every fetched row is by construction a private or +//! quarantined repo the caller may not read. The row's `created_at` leaks its +//! creation time and its `id` carries the owner's DID, so both halves are withheld +//! fields and the token must be CONFIDENTIAL, not merely tamper-evident: +//! +//! * AEAD-sealed (XChaCha20-Poly1305), never base64-of-plaintext and never +//! signed plaintext. Integrity is not confidentiality. +//! * A fresh `OsRng` nonce on EVERY seal. Under a stream cipher a repeated nonce +//! means repeated keystream, and an attacker who can force the node to seal a +//! position whose plaintext they know XORs two tokens and recovers a withheld +//! row's fields in full — strictly worse than emitting plaintext. +//! * FIXED-WIDTH plaintext. AEAD ciphertext is plaintext-length plus the tag, and +//! both halves of a scan position vary in length, so a variable encoding would +//! make token LENGTH a side channel for the sealed row (a short name under a +//! short owner vs a long one). Every token this module mints is byte-identical +//! in length. +//! * The canonical CID as associated data, so a token minted while scanning for +//! one CID does not authenticate when replayed against another. +//! +//! Every failure to open — wrong key, tampered bytes, wrong CID, expired, malformed +//! — returns the same `None`. The caller treats that as "no token" and starts at the +//! front, so no failure class is distinguishable and the token is no oracle. + +use base64::{engine::general_purpose::URL_SAFE_NO_PAD as B64URL, Engine}; +use chacha20poly1305::{ + aead::{Aead, KeyInit, OsRng, Payload}, + XChaCha20Poly1305, XNonce, +}; +use rand::RngCore; + +/// Keyset position of the last row a truncated scan fetched. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScanPosition { + /// The row's raw stored `created_at` text, the first half of the keyset cursor. + pub created_at_key: String, + /// The row's `id`, the tiebreaking half of the keyset cursor. + pub id: String, +} + +/// Plaintext version byte, so a future layout change is a clean open-failure +/// (treated as absent) rather than a misparse. +const VERSION: u8 = 1; + +/// Byte width each variable-length field is padded to. Both halves of a scan +/// position are stored at this width regardless of content, which is what keeps +/// every minted token the same length. A repo id is `/`, so 128 +/// clears a `did:key` z-base58 owner plus a long name with room to spare; anything +/// past it fails the seal loudly rather than silently truncating a cursor (a +/// truncated cursor would resume at the wrong row and skip coverage). +const FIELD_WIDTH: usize = 128; + +/// `version | created_len:u16 | created[FIELD_WIDTH] | id_len:u16 | id[FIELD_WIDTH] | expires:i64` +const PLAINTEXT_LEN: usize = 1 + 2 + FIELD_WIDTH + 2 + FIELD_WIDTH + 8; + +/// Nonce width for XChaCha20-Poly1305. +const NONCE_LEN: usize = 24; + +/// A fresh random 32-byte sealing key from the OS CSPRNG. +pub fn new_key() -> [u8; 32] { + let mut key = [0u8; 32]; + OsRng.fill_bytes(&mut key); + key +} + +// The two halves below are deliberately separate and adjacent. The framing pair owns +// "every token is the same length"; the AEAD pair owns "the contents are confidential +// and CID-bound". Keeping them apart is what lets each property be exercised — and +// broken — without disturbing the other. + +/// Encode a position into the FIXED-WIDTH plaintext: +/// `version | created_len:u16 | created[FIELD_WIDTH] | id_len:u16 | id[FIELD_WIDTH] | expires:i64` +/// +/// The padding is the point. AEAD ciphertext is plaintext-length plus the tag, and both +/// halves of a scan position vary in length, so a length-prefixed encoding with no +/// padding would make token LENGTH a side channel for the sealed row. +fn encode_position(pos: &ScanPosition, expires_at_unix: i64) -> anyhow::Result> { + let mut out = vec![0u8; PLAINTEXT_LEN]; + out[0] = VERSION; + let mut at = 1; + for field in [pos.created_at_key.as_bytes(), pos.id.as_bytes()] { + if field.len() > FIELD_WIDTH { + // Loud rather than truncating: a clipped cursor resumes at the wrong row and + // silently skips coverage, which is the availability half of the bug this + // token exists to fix. + anyhow::bail!( + "scan token field is {} bytes, over the {FIELD_WIDTH}-byte fixed width", + field.len() + ); + } + out[at..at + 2].copy_from_slice(&(field.len() as u16).to_le_bytes()); + at += 2; + out[at..at + field.len()].copy_from_slice(field); + at += FIELD_WIDTH; + } + out[at..at + 8].copy_from_slice(&expires_at_unix.to_le_bytes()); + Ok(out) +} + +/// Decode what [`encode_position`] wrote. `None` on any structural mismatch. +fn decode_position(bytes: &[u8]) -> Option<(ScanPosition, i64)> { + if bytes.len() != PLAINTEXT_LEN || bytes[0] != VERSION { + return None; + } + let mut at = 1; + let mut fields = [const { String::new() }; 2]; + for slot in fields.iter_mut() { + let len = u16::from_le_bytes([bytes[at], bytes[at + 1]]) as usize; + at += 2; + if len > FIELD_WIDTH { + return None; + } + *slot = String::from_utf8(bytes[at..at + len].to_vec()).ok()?; + at += FIELD_WIDTH; + } + let expires_at = i64::from_le_bytes(bytes[at..at + 8].try_into().ok()?); + let [created_at_key, id] = fields; + Some((ScanPosition { created_at_key, id }, expires_at)) +} + +/// AEAD-seal `plaintext` under `key`, bound to `cid`, framed as `nonce || ciphertext`. +fn seal_bytes(key: &[u8; 32], cid: &str, plaintext: &[u8]) -> anyhow::Result> { + let cipher = XChaCha20Poly1305::new_from_slice(key) + .map_err(|e| anyhow::anyhow!("scan token key: {e}"))?; + // A FRESH nonce per seal, from the OS CSPRNG. Under a stream cipher a repeated nonce + // repeats the keystream, and two tokens sealed under one nonce XOR to the difference + // of their plaintexts — which recovers a withheld row in full when the attacker can + // force one of the two positions. This draw is the property the whole confidentiality + // claim rests on. + let mut nonce = [0u8; NONCE_LEN]; + OsRng.fill_bytes(&mut nonce); + let sealed = cipher + .encrypt( + XNonce::from_slice(&nonce), + Payload { + msg: plaintext, + // The canonical CID as associated data: a token minted while scanning for + // one CID does not authenticate against another, so it cannot be replayed + // to seed a different scan. + aad: cid.as_bytes(), + }, + ) + .map_err(|e| anyhow::anyhow!("scan token seal: {e}"))?; + let mut out = Vec::with_capacity(NONCE_LEN + sealed.len()); + out.extend_from_slice(&nonce); + out.extend_from_slice(&sealed); + Ok(out) +} + +/// Open what [`seal_bytes`] framed. `None` on any failure, including a wrong `cid`. +fn open_bytes(key: &[u8; 32], cid: &str, raw: &[u8]) -> Option> { + if raw.len() <= NONCE_LEN + 16 { + return None; + } + let (nonce, sealed) = raw.split_at(NONCE_LEN); + let cipher = XChaCha20Poly1305::new_from_slice(key).ok()?; + cipher + .decrypt( + XNonce::from_slice(nonce), + Payload { + msg: sealed, + aad: cid.as_bytes(), + }, + ) + .ok() +} + +/// Seal `pos` under `key`, bound to `cid`, expiring at `expires_at_unix`. +/// +/// Returns the base64url (no pad) token. Errors only when a field exceeds +/// [`FIELD_WIDTH`] or the AEAD itself fails — never silently truncates. +pub fn seal_scan_token( + key: &[u8; 32], + cid: &str, + pos: &ScanPosition, + expires_at_unix: i64, +) -> anyhow::Result { + let plaintext = encode_position(pos, expires_at_unix)?; + Ok(B64URL.encode(seal_bytes(key, cid, &plaintext)?)) +} + +/// Open a token minted by [`seal_scan_token`] under the same key and CID. +/// +/// `None` for every failure class alike (wrong key, tampered, foreign CID, expired, +/// malformed, wrong version), so the caller can treat all of them as "absent" without +/// leaking which one occurred. +pub fn open_scan_token( + key: &[u8; 32], + cid: &str, + token: &str, + now_unix: i64, +) -> Option { + let raw = B64URL.decode(token).ok()?; + let plaintext = open_bytes(key, cid, &raw)?; + let (pos, expires_at) = decode_position(&plaintext)?; + if now_unix >= expires_at { + return None; + } + Some(pos) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn pos(created: &str, id: &str) -> ScanPosition { + ScanPosition { + created_at_key: created.to_string(), + id: id.to_string(), + } + } + + #[test] + fn round_trips_under_the_same_key_and_cid() { + let key = new_key(); + let p = pos("2020-01-01T00:00:03+00:00", "z6MkOwner/private-repo"); + let t = seal_scan_token(&key, "bafkcid", &p, 1 << 40).unwrap(); + assert_eq!(open_scan_token(&key, "bafkcid", &t, 0), Some(p)); + } + + #[test] + fn every_failure_class_opens_to_none() { + let key = new_key(); + let other = new_key(); + let p = pos("2020-01-01T00:00:03+00:00", "z6MkOwner/private-repo"); + let t = seal_scan_token(&key, "bafkcid", &p, 1 << 40).unwrap(); + + assert_eq!(open_scan_token(&other, "bafkcid", &t, 0), None, "wrong key"); + assert_eq!( + open_scan_token(&key, "bafkOTHER", &t, 0), + None, + "foreign CID" + ); + assert_eq!( + open_scan_token(&key, "bafkcid", &t, 1 << 41), + None, + "expired" + ); + assert_eq!(open_scan_token(&key, "bafkcid", "!!not b64", 0), None); + assert_eq!(open_scan_token(&key, "bafkcid", "", 0), None); + let mut flipped: Vec = t.bytes().collect(); + let last = flipped.len() - 1; + flipped[last] = if flipped[last] == b'A' { b'B' } else { b'A' }; + assert_eq!( + open_scan_token(&key, "bafkcid", &String::from_utf8(flipped).unwrap(), 0), + None, + "tampered" + ); + } + + #[test] + fn a_field_over_the_fixed_width_fails_loudly() { + let key = new_key(); + let p = pos("2020-01-01T00:00:03+00:00", &"x".repeat(FIELD_WIDTH + 1)); + assert!( + seal_scan_token(&key, "bafkcid", &p, 1 << 40).is_err(), + "an over-wide field must fail the seal, never be truncated into a cursor \ + that resumes at the wrong row" + ); + } +} diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f9d9b855..f1c05b69 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -86,6 +86,45 @@ pub(crate) const MAX_LEGACY_PROBES_PER_REQUEST: u32 = 256; /// sibling caps. pub(crate) const LEGACY_SCAN_PAGE_ROWS: usize = 128; +/// Hard per-request ceiling on how many repo ROWS the legacy scan's pager may fetch +/// (#173 round 13, F2, INV-10). The probe ceiling above only starts counting once a +/// probe runs, and the two denial classes that dominate a hostile inventory — +/// quarantine and a root-scope visibility deny — return before either `walk.probes` +/// or `walk.visits` increments. So an all-quarantined or all-root-denying node paged +/// through its ENTIRE repo table at zero probes, anonymously, retaining every row and +/// rule set, while holding one of the scarce global walk permits for up to the whole +/// request budget. This ceiling is what the DB-facing selection actually stops on. +/// +/// Reaching a holder buried past the ceiling costs `ceil(repos / ceiling) + 1` +/// token-echoing retries: a truncated scan sheds the retryable 503 with a sealed +/// continuation (`ScanPosition`), and the caller echoes it as `?scan=` to resume +/// exactly where the previous page stopped. No server-side scan state exists, so +/// concurrent callers cannot advance or reset each other's ladder. +/// +/// Above roughly `ceiling * (work-budget page term)` rows the bound's total page cost +/// exceeds one work-budget window, so a caller laddering a very large inventory will +/// meet the per-IP page toll before the end and resume after their bucket refills. +/// +/// Tuning DOWN has a cost worth stating: token presence is a coarse inventory-size +/// oracle. A ceiling truncation emits a token; a wrapped scan does not, so laddering +/// until the `scan-wrapped` taint tells an anonymous caller the node's TOTAL repo +/// count — private and quarantined rows included — to within one ceiling. At the 2048 +/// default that is tolled and coarse; it sharpens as the ceiling is lowered. +/// +/// Tunable via `GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS` / `AppState`. +pub(crate) const MAX_LEGACY_SCAN_ROWS_PER_REQUEST: usize = 2048; + +/// Hard per-request ceiling on how many visibility RULES the legacy scan's pager may +/// retain (#173 round 13, F2, INV-10). The row ceiling bounds the row count but not +/// the memory each row drags in: `fetch_next_page` keeps every fetched page's rules in +/// `LegacyScanPager::rules` for the whole request (a later oid candidate re-reads them +/// rather than re-querying), so a node whose repos each carry hundreds of path-scoped +/// rules is retained-memory-unbounded at a row count well under the row ceiling. +/// Counting the retained rules and stopping on them is the second half of the same +/// bound. Not an operator knob: it is a memory guard, not a reach/coverage tradeoff, +/// and 8192 is four rules per row at the default row ceiling. +pub(crate) const MAX_LEGACY_SCAN_RULES_PER_REQUEST: usize = 8192; + /// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` buffers and serves /// (#173 round 8, F6, INV-10). The serve reads via a blocking `git cat-file` and /// buffers the whole object; unbounded, a large public blob (enumerable from the pins @@ -124,6 +163,17 @@ struct LegacyScanPager { cursor: Option<(String, String)>, /// Set once a short page proves no rows remain after the cursor. exhausted: bool, + /// True when `cursor` was seeded from a caller-supplied continuation token rather + /// than starting at the front. Half of the `"scan-wrapped"` condition: absence is + /// only ever proven over `[start, end)`, so a resumed scan that runs off the end + /// has NOT covered `[front, start)` and must never reach the definitive 404. + resumed: bool, + /// Rows fetched THIS request, the quantity the row ceiling bounds. Distinct from + /// `rows.len()`, which is the same number today but would silently stop tracking + /// the DB-facing cost if the pager ever dropped gated rows. + fetched_rows: usize, + /// Visibility rules retained this request, the quantity the rules ceiling bounds. + fetched_rules: usize, } impl LegacyScanPager { @@ -174,6 +224,7 @@ impl LegacyScanPager { }; #[cfg(test)] note_scan_rows(page.len()); + self.fetched_rows += page.len(); if page.len() < state.ipfs_legacy_scan_page_rows { self.exhausted = true; } @@ -200,12 +251,29 @@ impl LegacyScanPager { return Err(budget_shed()); } }; + self.fetched_rules += rules.values().map(Vec::len).sum::(); self.rules.extend(rules); self.rows.extend(page); Ok(()) } } +/// Query string of `GET /ipfs/{cid}`. +#[derive(serde::Deserialize)] +pub struct ScanQuery { + /// Sealed continuation from a previous truncated scan's 503 body. Opened with the + /// node's per-boot key and the request's canonical CID as associated data; ANY + /// failure (undecryptable, tampered, expired, malformed, minted for another CID) is + /// treated as absent and the scan starts at the front, identically and silently, so + /// the token is no oracle. + scan: Option, +} + +/// How long a continuation stays usable. Long enough for a caller to walk a ladder at a +/// human pace and to ride out a work-bucket throttle; short enough that a leaked token +/// stops being a valid scan seed quickly. An expired token is simply absent. +const SCAN_TOKEN_TTL_SECS: i64 = 3600; + /// GET /ipfs/{cid} /// /// Resolve the CIDv1 to its git oid via the `pinned_cids` table, then search all @@ -280,6 +348,7 @@ struct WalkAdmission { pub async fn get_by_cid( Path(cid_str): Path, + axum::extract::Query(scan_query): axum::extract::Query, State(state): State, crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, headers: HeaderMap, @@ -457,6 +526,27 @@ pub async fn get_by_cid( // cursor and its fetched rows are accounted per REQUEST: a per-candidate pager // would restore the very fan-out the paging removes. let mut pager = LegacyScanPager::default(); + // Resume from the caller's sealed continuation, if they sent one that opens. The + // node holds NO scan state of its own: the position rides in the token, which is + // what keeps concurrent ladders from advancing or resetting each other. Every + // failure class — tampered, wrong key (a prior boot's), expired, malformed, minted + // for a different CID — lands on the same `None` and starts at the front, silently, + // so no probe distinguishes them (INV-13). + if let Some(token) = scan_query.scan.as_deref() { + if let Some(pos) = gitlawb_core::scan_token::open_scan_token( + &state.ipfs_scan_token_key, + &canonical_cid, + token, + chrono::Utc::now().timestamp(), + ) { + pager.cursor = Some((pos.created_at_key, pos.id)); + pager.resumed = true; + } + } + // Set when a ceiling truncates the scan, to the position the caller echoes back. + // Sealed at the tail rather than here so exactly one site mints a token and the + // wrap case can clear it in one place. + let mut scan_continuation: Option<(String, String)> = None; for sha256_hex in &oids { // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). @@ -678,15 +768,17 @@ pub async fn get_by_cid( // this is the check that keeps the DB-facing selection bounded, so // a one-probe request cannot pull the node's whole inventory. // - // Note what is deliberately NOT a stop condition: a page of pure - // denials. A quarantined row or a visibility deny costs no probe, - // so paging must continue past them or a public object buried - // behind many private repos would falsely 404. The budgets above, - // not a page count, are what bound that case. + // A page of pure denials is still NOT a hard stop into a 404: a + // quarantined row or a visibility deny costs no probe, so paging + // must continue past them or a public object buried behind many + // private repos would falsely 404. What bounds that case is not a + // denial count but the DB-facing ceilings just below, and every + // truncation they cause carries a continuation so the buried object + // stays reachable across requests (#173 round 13, F2). // - // Stopping here leaves every unread repo unproven, so it TAINTS: - // the tail sheds a retryable 503 naming the ceiling, never a - // definitive 404 (#173, F2). + // Stopping at any of these leaves every unread repo unproven, so + // each TAINTS: the tail sheds a retryable 503 naming the ceiling, + // never a definitive 404 (#173, F2). if walk.probes >= state.ipfs_max_legacy_probes { walk.taint("probe-ceiling"); break; @@ -695,6 +787,45 @@ pub async fn get_by_cid( walk.taint("visit-ceiling"); break; } + // Row ceiling (F2). The two checks above only bind once a probe or a + // visit has been spent, and the gate returns Skip on quarantine and + // on a root-scope deny BEFORE either counter moves — so an + // all-denying inventory paged the node's whole repo table at zero + // probes, anonymously, while holding a scarce walk permit. This is + // the check that actually stops that scan. + if pager.fetched_rows >= state.ipfs_max_legacy_scan_rows { + walk.taint("row-ceiling"); + scan_continuation = pager.cursor.clone(); + break; + } + // Rules ceiling: the row ceiling bounds rows, not the rules each row + // drags in, and the pager retains every fetched page's rules for the + // whole request. + if pager.fetched_rules >= state.ipfs_max_legacy_scan_rules { + walk.taint("rules-ceiling"); + scan_continuation = pager.cursor.clone(); + break; + } + // Page toll (F2). Every page is work bought by an anonymous caller, + // so it is charged to the per-IP WORK bucket — the same bucket the + // per-probe charge debits — immediately before the query it pays + // for. Without it a denial-only inventory could be re-paged for free + // by re-requesting, which is the across-request half of the same + // amplification. Reuses the `source_key` already resolved at + // admission; no resolvable key (a test oneshot with no peer or + // trusted header) skips the charge, exactly as the walk and probe + // brakes do. + // + // A spent bucket sets `throttled` and breaks WITHOUT tainting and + // WITHOUT a token: the caller's own bucket stopped them, their + // previous token still resumes them after it refills, and the tail + // renders the 429 when nothing else tainted. + if let Some(key) = &source_key { + if !state.ipfs_work_rate_limiter.check(key).await { + throttled = true; + break; + } + } pager .fetch_next_page(&state, request_deadline, &cid_str) .await?; @@ -731,6 +862,25 @@ pub async fn get_by_cid( } } + // A RESUMED scan that reached the end of the table has proven absence only over + // `[token, end)`; the rows before the token were never looked at this request, so + // the definitive 404 is not available and the honest answer is the retryable 503. + // + // The condition is evaluated HERE, on `pager.exhausted`, and deliberately not at any + // particular break site. That is what covers the degenerate zero-row resume: a token + // at or past the last row — which the row ceiling emits whenever the row count is an + // exact multiple of the ceiling, and which repo deletion between ladder steps also + // reaches — fetches an EMPTY short page, sets `exhausted`, and breaks without + // gating anything. An implementation keying this on having fetched a page passes + // every other case and turns exactly that incomplete search into a false 404. + // + // A wrapped scan emits NO continuation: there is nothing left to resume, and the + // absence of the token is what tells the caller their ladder is over. + if pager.resumed && pager.exhausted { + walk.taint("scan-wrapped"); + scan_continuation = None; + } + // Nothing served — four distinct tails, in precedence order: // 1. A candidate repo is persistently broken (a corrupt repo, a bad `.git/config`), // and that was the SOLE reason nothing served → terminal, non-retryable 500 @@ -745,7 +895,20 @@ pub async fn get_by_cid( // retryable, and explicitly NOT a definitive not-found (#173 F2). This outranks // the throttle: an incomplete search must not masquerade as a clean rate-limit // outcome. The message names the truncation sources so an operator can map the - // shed to the right knob or backend, and carries no object/OID/metadata. + // shed to the right knob or backend, and carries no object/OID/metadata. When a + // ceiling was the cut, the shed also carries the sealed continuation the caller + // echoes as `?scan=` to resume. + // + // ONE deliberate exception to that precedence (#173 round 13, F2): the legacy + // scan's PAGE toll breaks the pager WITHOUT tainting, so a request stopped only + // by its own spent work bucket falls through to the 429 below rather than the + // 503 here. The reason is that a 503 says "the node's search was cut short, + // retry" and invites an immediate retry straight back into the same empty + // bucket; a 429 names what actually stopped the caller and carries the honest + // wait. Their previously issued token is still valid, so the retry after the + // refill resumes rather than restarts. A request that tainted for any OTHER + // reason and then also ran its bucket dry still lands here, per the ordering + // as written. // 3. A walk-requiring candidate was skipped for a spent IP quota while the scan // otherwise completed → 429 (the brake bit; a cheaper copy was sought first). // 4. A full scan under the caps found nothing readable → opaque 404, uniform with @@ -756,10 +919,35 @@ pub async fn get_by_cid( )); } if !walk.truncated_by.is_empty() { - return Err(AppError::SearchIncomplete(format!( - "CID {cid_str} search incomplete ({}) — retry", - walk.truncated_by.join("+") - ))); + // Seal the continuation HERE, the single mint site. The position is the last + // row the pager FETCHED, and on a scan that served nothing every fetched row is + // by construction private or quarantined — so its `created_at` and its `id` + // (which carries the owner's DID) are withheld fields and the token must be + // confidential, not merely tamper-evident (INV-13). A seal failure is not fatal + // to the shed: drop the continuation and answer the plain 503, which degrades to + // the pre-token behaviour rather than turning a truncation into a 500. + let continuation = scan_continuation.and_then(|(created_at_key, id)| { + match gitlawb_core::scan_token::seal_scan_token( + &state.ipfs_scan_token_key, + &canonical_cid, + &gitlawb_core::scan_token::ScanPosition { created_at_key, id }, + chrono::Utc::now().timestamp() + SCAN_TOKEN_TTL_SECS, + ) { + Ok(token) => Some(token), + Err(e) => { + tracing::warn!(error = %e, "/ipfs could not seal a scan continuation; \ + shedding the truncation 503 without one"); + None + } + } + }); + return Err(AppError::SearchIncomplete { + message: format!( + "CID {cid_str} search incomplete ({}) — retry", + walk.truncated_by.join("+") + ), + continuation, + }); } if throttled { return Err(AppError::TooManyRequests( @@ -1831,6 +2019,139 @@ mod tests { "f2".repeat(32) } + /// Status plus decoded JSON body, for the F2 row-ceiling tests that assert on the + /// error code and the `continuation` field together. + async fn status_and_body(resp: axum::response::Response) -> (StatusCode, serde_json::Value) { + let status = resp.status(); + let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .expect("read body"); + let json = serde_json::from_slice(&bytes).unwrap_or_else( + |_| serde_json::json!({ "raw": String::from_utf8_lossy(&bytes).to_string() }), + ); + (status, json) + } + + /// The `continuation` token from a `search_incomplete` body, or `None`. + fn continuation_of(body: &serde_json::Value) -> Option { + body.get("continuation") + .and_then(|v| v.as_str()) + .map(str::to_string) + } + + /// Deterministic ascending `created_at` for the seeded-inventory fixtures. The + /// paged scan orders on the STORED `created_at` TEXT then `id`, so whole-second + /// stamps from one base keep text order and time order identical (a `to_rfc3339` + /// with sub-second digits on some rows and not others would not). + fn scan_order_stamp(i: usize) -> chrono::DateTime { + use chrono::TimeZone; + chrono::Utc + .with_ymd_and_hms(2020, 1, 1, 0, 0, 0) + .unwrap() + .checked_add_signed(chrono::Duration::seconds(i as i64)) + .expect("in-range stamp") + } + + /// Seed `n` PRIVATE repos owned by a foreign DID, in scan order, with `rules_each` + /// path-scoped rules apiece. An anonymous caller is denied at the root gate on every + /// one, and a root deny costs neither a probe nor a visit — which is exactly the + /// hole the row ceiling closes. Their `disk_path`s do not exist on purpose: if a + /// deny ever stopped short-circuiting, the missing-dir probe would taint the scan + /// with a different source and the tests' taint assertions would catch it. + async fn seed_root_denying_repos( + state: &crate::state::AppState, + prefix: &str, + n: usize, + rules_each: usize, + ) { + let owner = "did:key:z6MkF2RowCeilingOwnerAAAAAAAAAAAAAAAAAAA"; + for i in 0..n { + let at = scan_order_stamp(i); + let id = format!("{prefix}-{i:04}"); + state + .db + .create_repo(&crate::db::RepoRecord { + id: id.clone(), + name: format!("{prefix}-{i:04}"), + owner_did: owner.to_string(), + description: None, + is_public: false, + default_branch: "main".to_string(), + created_at: at, + updated_at: at, + disk_path: format!("/nonexistent/{prefix}-{i:04}"), + forked_from: None, + machine_id: None, + }) + .await + .expect("seed a root-denying repo"); + for r in 0..rules_each { + state + .db + .set_visibility_rule( + &id, + &format!("withheld-{r}/**"), + crate::db::VisibilityMode::B, + &["did:key:z6MkU3NotTheCallerBBBBBBBBBBBBBBBBBBBBBB".to_string()], + owner, + ) + .await + .expect("seed a visibility rule"); + } + } + } + + /// Seed `n` QUARANTINED mirror rows in scan order. Quarantine is the other denial + /// class that returns from the gate before a probe or a visit is spent, so it drives + /// the same unbounded pager the private-repo fixture does. + async fn seed_quarantined_repos(state: &crate::state::AppState, prefix: &str, n: usize) { + for i in 0..n { + state + .db + .upsert_mirror_repo( + "z6quarantine", + &format!("{prefix}-{i:04}"), + &format!("/nonexistent/{prefix}-{i:04}"), + None, + true, + ) + .await + .expect("seed a quarantined mirror row"); + } + } + + /// A GET carrying an optional `?scan=` continuation token. + fn get_cid_scan(cid: &str, peer: Option, scan: Option<&str>) -> Request { + let uri = match scan { + Some(t) => format!("/ipfs/{cid}?scan={}", urlencode(t)), + None => format!("/ipfs/{cid}"), + }; + let mut req = Request::builder() + .method(Method::GET) + .uri(uri) + .body(Body::empty()) + .unwrap(); + if let Some(p) = peer { + req.extensions_mut().insert(ConnectInfo(p)); + } + req + } + + /// Percent-encode the few characters base64url tokens cannot contain but a hostile + /// or tampered token can. Keeps the invalid-token probes honest: a raw `+` in a + /// query string decodes to a space, which would make a tamper test pass for the + /// wrong reason. + fn urlencode(s: &str) -> String { + s.bytes() + .map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + (b as char).to_string() + } + _ => format!("%{b:02X}"), + }) + .collect() + } + /// Register a LEGACY (NULL-provenance) `pinned_cids` row and return its CID. /// /// The scan-verdict tests below predate the CID index (#173): they drove a bare @@ -2877,6 +3198,1066 @@ mod tests { ); } + // ---------------------------------------------------------------------------- + // #173 round 13, F2: the legacy scan's ROW ceiling, its caller-carried + // continuation token, and the per-page toll. + // + // The hole: the pager bought another page unless `walk.probes` or `walk.visits` + // was exhausted, but the gate returns Skip on quarantine and on a root-scope + // visibility deny BEFORE either counter increments. An all-quarantined or + // all-root-denying inventory therefore paged the node's entire repo table at zero + // probes, anonymously, retaining every row and rule set, while holding one of the + // scarce global walk permits for up to the whole request budget. + // ---------------------------------------------------------------------------- + + /// Scenario 1: an all-root-denied inventory stops at the row ceiling. + /// + /// Every seeded repo is private and the caller is anonymous, so each row is a root + /// deny: no probe, no visit, and pre-fix nothing that could stop the pager. The + /// scan must stop at the ceiling, taint (so the tail is the retryable 503, never a + /// false 404), free the walk permit, and hand back a continuation token. + /// + /// MUTATION A (RED): delete the row-ceiling check and `scan_rows()` reads the whole + /// seeded inventory instead of one ceiling's worth. + #[sqlx::test] + async fn get_by_cid_denial_only_scan_stops_at_row_ceiling(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 4; + seed_root_denying_repos(&state, "deny", 12, 0).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let walk_pool = state.git_ipfs_walk_semaphore.clone(); + let free_before = walk_pool.available_permits(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.140:5000".parse().unwrap(); + + crate::api::ipfs::reset_scan_rows(); + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + + // The row COUNT first: it is the cost this ceiling exists to bound, and a + // status-first ordering would attribute a missing ceiling to the tail instead. + let rows = crate::api::ipfs::scan_rows(); + assert!( + rows <= 4 + 2, + "the ceiling (4) bounds the DB-facing selection to at most one page (2) of \ + overshoot; a denial-only inventory must not page the whole table. Read {rows} \ + of 12 seeded rows" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a scan cut short at the row ceiling left rows unproven, so the honest tail \ + is the retryable 503, never a definitive 404; got body {body}" + ); + assert_eq!( + body["error"], "search_incomplete", + "the shed must name the incomplete search: {body}" + ); + assert!( + continuation_of(&body).is_some(), + "a ceiling truncation must hand back a continuation so a holder past the \ + ceiling is still reachable: {body}" + ); + assert_eq!( + walk_pool.available_permits(), + free_before, + "the shed must free the scarce walk admission, not hold it for the request budget" + ); + + // The follow-up is ADMITTED: the shed released the walk permit rather than + // parking it, so the next caller is not capacity-503'd behind it. + let (status, _) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the follow-up must be admitted and reach the same truncation verdict, not \ + shed at capacity behind a held permit" + ); + } + + /// Scenario 2: an all-QUARANTINED inventory, same contract. Quarantine is the other + /// denial class that returns from the gate before a probe or a visit is spent, so a + /// ceiling keyed on either counter would miss it entirely. + /// + /// MUTATION A (RED): as scenario 1. + #[sqlx::test] + async fn get_by_cid_quarantined_only_scan_stops_at_row_ceiling(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 4; + seed_quarantined_repos(&state, "quar", 12).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let peer: SocketAddr = "203.0.113.141:5000".parse().unwrap(); + crate::api::ipfs::reset_scan_rows(); + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a quarantined-only inventory truncates at the ceiling like a denied one: {body}" + ); + assert_eq!(body["error"], "search_incomplete", "{body}"); + let rows = crate::api::ipfs::scan_rows(); + assert!( + rows <= 4 + 2, + "quarantine costs neither a probe nor a visit, so only the ROW ceiling can \ + stop this pager. Read {rows} of 12 seeded rows" + ); + assert!( + continuation_of(&body).is_some(), + "the quarantined-inventory truncation carries a continuation too: {body}" + ); + } + + /// Scenario 3 (must-not): a buried PUBLIC row inside the ceiling still serves. The + /// ceiling bounds the search; it must never convert reachable content into a shed. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_public_row_inside_row_ceiling_still_serves(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + // Ceiling comfortably ABOVE the inventory: nothing may truncate here. + state.ipfs_max_legacy_scan_rows = 64; + + seed_root_denying_repos(&state, "buried", 5, 0).await; + // Seeded last, and `upsert_mirror_repo` stamps `now`, so this row sorts after + // every 2020-stamped denial row and is genuinely reached last. + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6inside", + "holder", + b"inside ceiling\n", + ) + .await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let peer: SocketAddr = "203.0.113.142:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::OK, + "a public holder inside the ceiling must serve; a ceiling that sheds \ + reachable content is worse than the unbounded scan it replaced" + ); + } + + /// Scenario 4 (must-not): no ceiling ever produces a 404. + /// + /// Three legs against one genuinely-absent object over 5 denial rows at ceiling 2: + /// * a front-started truncated scan is 503 `search_incomplete` WITH a token; + /// * a token-resumed scan that reaches the table end taints `scan-wrapped` and + /// emits NO token (absence was proven only over `[start, end)`); + /// * only a front-started scan that exhausts under every ceiling reaches the 404. + /// + /// MUTATION B (RED): replace taint-and-break with a bare `break` and the first leg + /// becomes the 404 tail. + #[sqlx::test] + async fn get_by_cid_row_ceiling_never_returns_404(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 2; + seed_root_denying_repos(&state, "no404", 5, 0).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let router = ipfs_router(state.clone()); + let peer: SocketAddr = "203.0.113.143:5000".parse().unwrap(); + + // Leg 1: front-started truncation. + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a truncated scan is never a 404: {body}" + ); + assert_eq!(body["error"], "search_incomplete", "{body}"); + let mut token = continuation_of(&body).expect("leg 1 must emit a continuation"); + + // Leg 2: ladder to the end. 5 rows at ceiling 2 truncates twice, then the third + // resume reads the short final page and WRAPS. + let mut wrapped = None; + for step in 0..6 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "every rung of the ladder over an absent object is a retryable 503, \ + never a 404 (step {step}): {body}" + ); + match continuation_of(&body) { + Some(next) => token = next, + None => { + wrapped = Some(body); + break; + } + } + } + let wrapped = wrapped.expect("the ladder must reach the table end within its bound"); + assert!( + wrapped["message"] + .as_str() + .unwrap_or_default() + .contains("scan-wrapped"), + "a resumed scan that reaches the end must taint scan-wrapped, so absence \ + proven only over [start, end) is never reported as a definitive 404: {wrapped}" + ); + assert!( + continuation_of(&wrapped).is_none(), + "a wrapped scan emits NO token — there is nothing left to resume: {wrapped}" + ); + + // Leg 3: the 404 tail stays reachable for a front-started scan that exhausts + // under every ceiling. + let mut wide = state.clone(); + wide.ipfs_max_legacy_scan_rows = 1000; + let (status, body) = status_and_body( + ipfs_router(wide) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "a front-started scan that exhausts under every ceiling still gets the \ + definitive 404; a ceiling that swallowed it would make every miss retryable \ + forever: {body}" + ); + } + + /// Scenario 5: a holder buried PAST the ceiling becomes servable within the stated + /// bound by echoing tokens, under the PRODUCTION toll at its raised derived floor. + /// + /// Ceiling 4 over 10 denial rows with the public holder behind them: the bound is + /// `ceil(10 / 4) + 1 = 4` requests. Every intermediate response is the retryable + /// 503 with a token, and no 429 interrupts the ladder — which is what the floor fix + /// pins. The work bucket is sized to the DERIVED floor of a config whose page term + /// dominates (probe knob 1, row knob 896 = 7 pages, so floor = 8); under the old + /// floor (`max(route, probes)` = 1) the very first page would 429. + /// + /// The floor is 8 rather than the honest ladder's exact cost (6 pages + 1 probe = 7) + /// on purpose. A ladder that never resumes re-pages from the front every request and + /// costs 8, so at a bucket of 7 mutation C would trip the 429 guard one step before + /// the reach guard and its RED would be attributed to the toll rather than to the + /// missing continuation. One token of headroom keeps each guard reporting its own + /// property. + /// + /// MUTATION C (RED): emit the token but never open it on the way in, and the ladder + /// restarts at the front every time so the 200 never arrives. + /// + /// This test has NO pre-fix RED, and that is by design rather than an omission. + /// Mutation A (delete the row ceiling) must leave it GREEN, which means its + /// assertions have to tolerate the holder being served on the very first request — + /// exactly what an unbounded scan does. So the pre-fix head passes it. Its + /// load-bearing proof is mutation C, its designated mutant: C keeps the ceiling and + /// keeps minting tokens but never honours one, which is the only shape that makes + /// the holder permanently unservable. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_holder_past_scan_ceiling_serves_via_token_ladder(pool: sqlx::PgPool) { + use crate::state::AppState; + use clap::Parser; + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 4; + + // The production toll, sized exactly at its derived floor. The row knob here + // sizes the FLOOR (it reads the production 128-row page size); the ceiling the + // scan actually enforces is the AppState seam above, as with page rows. + let cfg = crate::config::Config::parse_from([ + "gitlawb-node", + "--ipfs-rate-limit", + "1", + "--ipfs-max-legacy-probes", + "1", + "--ipfs-max-legacy-scan-rows", + "896", + ]); + let floor = AppState::ipfs_work_budget(&cfg); + assert_eq!( + floor, 8, + "fixture precondition: 1 probe + 896/128 = 7 pages" + ); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(floor, std::time::Duration::from_secs(3600)); + + seed_root_denying_repos(&state, "ladder", 10, 0).await; + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6ladder", + "holder", + b"past the ceiling\n", + ) + .await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.144:5000".parse().unwrap(); + let bound = 10usize.div_ceil(4) + 1; + + let mut token: Option = None; + let mut served_at = None; + for step in 1..=bound { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + assert_ne!( + status, + StatusCode::TOO_MANY_REQUESTS, + "no 429 may interrupt an honest caller's ladder at step {step}: the work \ + floor must fit a full deep scan's page toll, or the reach bound is a \ + promise the toll breaks: {body}" + ); + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + assert_eq!(body["error"], "search_incomplete", "{body}"); + token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")), + ); + } + assert!( + served_at.is_some(), + "a holder past the ceiling must be served within ceil(10/4)+1 = {bound} \ + token-echoing requests, or the ceiling has made it permanently unservable" + ); + } + + /// Scenario 6: the page toll accumulates ACROSS requests. + /// + /// Every page the scan buys is charged to the caller's per-IP work bucket, so a + /// denial-only inventory cannot be re-paged for free by re-requesting. A bucket + /// sized to 4 pages admits four requests' worth of paging and then sheds the fifth + /// with 429 — buying NO page (the `preload_queries()` count stalls) and carrying NO + /// token. The caller's PREVIOUS token still resumes them once the bucket refills. + /// + /// MUTATION D (RED): drop the page toll and the pages are free again — the fifth + /// request buys its page and never 429s. + #[sqlx::test] + async fn get_by_cid_denial_only_requests_throttle_across_requests(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 2; + // Four pages of allowance: above the derived floor's page term for this fixture + // and still small enough that a handful of requests exhausts it. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(4, std::time::Duration::from_secs(3600)); + seed_root_denying_repos(&state, "toll", 20, 0).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let router = ipfs_router(state.clone()); + let peer: SocketAddr = "203.0.113.145:5000".parse().unwrap(); + + crate::api::ipfs::reset_preload_queries(); + let mut last_token = None; + for step in 1..=4 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "request {step} is within the bucket and must buy its page: {body}" + ); + last_token = continuation_of(&body); + } + let last_token = last_token.expect("a tolled-but-admitted request still emits a token"); + let pages_before = crate::api::ipfs::preload_queries(); + + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::TOO_MANY_REQUESTS, + "a spent work bucket must brake the next denial-only request with 429 \ + rather than sell it another page: {body}" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + pages_before, + "and the braked request must buy NO page — a 429 that still paged would \ + leave the amplification exactly where it was" + ); + assert!( + continuation_of(&body).is_none(), + "the 429 carries no token: the caller's own bucket, not the node's search, \ + stopped them, and their previous token is still valid: {body}" + ); + + // Bucket refilled (a fresh limiter is the window elapsing). The token the caller + // already holds still resumes them — the throttle cost them a page, not their + // place in the ladder. + let mut refilled = state.clone(); + refilled.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(4, std::time::Duration::from_secs(3600)); + crate::api::ipfs::reset_scan_rows(); + let (status, body) = status_and_body( + ipfs_router(refilled) + .oneshot(get_cid_scan(&cid, Some(peer), Some(&last_token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the previously issued token must still resume after a refill: {body}" + ); + assert_eq!( + crate::api::ipfs::scan_rows(), + 2, + "and it must resume at the sealed position — one ceiling's worth of rows \ + read, not a restart at the front" + ); + } + + /// Scenario 7: the RULES ceiling. The row ceiling bounds the row count but not the + /// memory each row drags in: the pager retains every fetched page's rules for the + /// whole request. A window of rule-heavy repos must taint at the rules ceiling with + /// the row count still well under the row ceiling, on the same 503-with-token + /// contract. + #[sqlx::test] + async fn get_by_cid_rules_ceiling_stops_scan_with_token(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + // Rows are NOT the binding ceiling here. + state.ipfs_max_legacy_scan_rows = 1000; + state.ipfs_max_legacy_scan_rules = 3; + seed_root_denying_repos(&state, "rules", 8, 2).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let peer: SocketAddr = "203.0.113.146:5000".parse().unwrap(); + crate::api::ipfs::reset_scan_rows(); + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a rules-ceiling truncation sheds the same retryable 503: {body}" + ); + assert!( + body["message"] + .as_str() + .unwrap_or_default() + .contains("rules-ceiling"), + "the shed must name the rules ceiling so an operator can tell it from a row \ + truncation: {body}" + ); + let rows = crate::api::ipfs::scan_rows(); + assert!( + rows < 1000, + "the rules ceiling must fire with rows still under the row ceiling, or it is \ + not the guard being exercised; read {rows} rows" + ); + assert!( + continuation_of(&body).is_some(), + "a rules truncation carries a continuation too: {body}" + ); + } + + /// Scenario 8: interleaved callers stay isolated. Two source keys alternate + /// token-echoing ladders against the same denial-heavy inventory with the holder + /// past the ceiling; each must reach its own 200 within its own bound. + /// + /// Isolation is STRUCTURAL under this design — each ladder's entire state rides in + /// its own tokens and the node holds none — so this is the executed confirmation + /// rather than a mutant target. It is what rules out the rejected designs: a + /// node-global persisted cursor lets these two advance each other's window, and a + /// per-caller server-side map lets one evict the other. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_interleaved_callers_each_reach_their_holder(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 4; + // The production toll, generous enough that neither caller's ladder is braked; + // this scenario is about isolation, not the toll. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(600, std::time::Duration::from_secs(3600)); + + seed_root_denying_repos(&state, "interleave", 10, 0).await; + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6interleave", + "holder", + b"shared holder\n", + ) + .await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let router = ipfs_router(state); + let peers: [SocketAddr; 2] = [ + "203.0.113.147:5000".parse().unwrap(), + "203.0.113.148:5000".parse().unwrap(), + ]; + let bound = 10usize.div_ceil(4) + 1; + let mut tokens: [Option; 2] = [None, None]; + let mut served = [None, None]; + + for step in 1..=bound { + for (i, peer) in peers.iter().enumerate() { + if served[i].is_some() { + continue; + } + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(*peer), tokens[i].as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served[i] = Some(step); + continue; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "caller {i} rung {step} must be the retryable 503: {body}" + ); + tokens[i] = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("caller {i} rung {step} needs a token: {body}")), + ); + } + } + + assert!( + served[0].is_some() && served[1].is_some(), + "both interleaved callers must reach the holder within their own bound of \ + {bound}; got {served:?}. A shared server-side cursor would let one caller's \ + progress skip the other's coverage" + ); + } + + /// Seed `n` PRIVATE repos whose id, owner DID, and `created_at` are all + /// high-entropy MARKERS, so a substring search over an emitted token is a real + /// test. Returns the markers in scan order. + async fn seed_marked_withheld_repos( + state: &crate::state::AppState, + n: usize, + ) -> Vec<(String, String, String)> { + const OWNER: &str = "did:key:z6MkWithheldOwnerMarkerQQQQQQQQQQQQQQQQ"; + let mut out = Vec::new(); + for i in 0..n { + let at = scan_order_stamp(i); + let id = format!("marker-repo-XZXZ{i:04}"); + // Every other row is a quarantined mirror instead of a private repo, so + // both withholding classes sit in the window the token is minted from. + if i % 2 == 1 { + state + .db + .upsert_mirror_repo(OWNER, &id, &format!("/nonexistent/{id}"), None, true) + .await + .expect("seed a quarantined marker row"); + // `upsert_mirror_repo` stamps `now` and derives its own id, so re-read + // the row the scan will actually see. + let rec = state + .db + .get_repo(OWNER, &id) + .await + .unwrap() + .expect("the quarantined marker row exists"); + out.push((rec.id, OWNER.to_string(), rec.created_at.to_rfc3339())); + continue; + } + state + .db + .create_repo(&crate::db::RepoRecord { + id: id.clone(), + name: id.clone(), + owner_did: OWNER.to_string(), + description: None, + is_public: false, + default_branch: "main".to_string(), + created_at: at, + updated_at: at, + disk_path: format!("/nonexistent/{id}"), + forked_from: None, + machine_id: None, + }) + .await + .expect("seed a private marker row"); + out.push((id, OWNER.to_string(), at.to_rfc3339())); + } + out + } + + /// Scenario 9, the INV-13 guard: the emitted continuation leaks no withheld field. + /// + /// A denial-only scan fetches nothing BUT withheld rows, so the row its token seals + /// is by construction a private or quarantined repo the caller may not read. Its + /// `created_at` leaks a hidden repo's creation time and its `id` carries the owner's + /// DID. Base64 is transport, not confidentiality — this is the exact shape #134 + /// shipped and INV-13 records — so the token must be AEAD-SEALED. + /// + /// The fixture is arranged so the row at the truncation boundary (the row the token + /// seals) IS one of the poisoned withheld repos. Stated because it is load-bearing: + /// a future edit seeding a READABLE repo at the boundary would leave mutation E + /// green and this guard would silently stop proving anything. + /// + /// The last assertion is the one the substring checks structurally cannot make. + /// AEAD ciphertext is plaintext-length plus the tag, and both halves of a scan + /// position vary in length, so without fixed-width padding the token LENGTH is a + /// side channel for the sealed row. + /// + /// MUTATION E (RED): seal by base64-of-plaintext and the markers decode straight out. + /// MUTATION G (RED): drop the fixed-width padding and the two lengths diverge. + /// + /// Like the two token guards below it, this has no pre-fix RED: its assertions call + /// `seal_scan_token` / `open_scan_token`, which do not exist on the pre-fix head, so + /// the only failure available there is a compile error. Mutations E and G are its + /// REDs, and each injects precisely the encoding INV-13 forbids rather than merely + /// removing the code, which is the stronger observation. + #[sqlx::test] + async fn scan_token_leaks_no_withheld_fields(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 2; + let markers = seed_marked_withheld_repos(&state, 6).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let key = state.ipfs_scan_token_key.clone(); + + let peer: SocketAddr = "203.0.113.150:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(); + let raw_body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .expect("read body"); + let body_text = String::from_utf8_lossy(&raw_body).to_string(); + let body: serde_json::Value = serde_json::from_slice(&raw_body).expect("json body"); + let token = continuation_of(&body).expect("the truncation must emit a token"); + + // Fixture precondition, on the FIXTURE rather than on the token: every seeded + // row is withheld (private or quarantined) and the scan stopped after exactly + // one ceiling's worth, so the row the token seals is a withheld row. Stated + // without opening the token so the leak assertions below are what fires when the + // seal is replaced by an encoding, rather than a precondition panic. + assert_eq!( + crate::api::ipfs::scan_rows(), + 2, + "the truncation boundary must sit inside the seeded withheld window" + ); + + let decoded = base64::Engine::decode( + &base64::engine::general_purpose::URL_SAFE_NO_PAD, + token.as_bytes(), + ) + .expect("the token is base64url"); + let decoded_text = String::from_utf8_lossy(&decoded).to_string(); + + for (id, owner, created) in &markers { + for (what, marker) in [ + ("repo id", id), + ("owner did", owner), + ("created_at", created), + ] { + assert!( + !body_text.contains(marker.as_str()), + "the response body must not carry a withheld repo's {what} ({marker}): \ + {body_text}" + ); + assert!( + !decoded_text.contains(marker.as_str()), + "the token's DECODED bytes must not carry a withheld repo's {what} \ + ({marker}) — base64 is transport, not confidentiality (INV-13)" + ); + assert!( + decoded + .windows(marker.len()) + .all(|w| w != marker.as_bytes()), + "the token's raw bytes must not carry a withheld repo's {what} ({marker})" + ); + } + } + + // And the row it actually seals IS one of the poisoned withheld rows — checked + // after the leak assertions so a broken seal is reported as a leak, not as a + // fixture failure. Load-bearing: seeding a READABLE repo at the boundary would + // leave mutation E green and this whole guard would stop proving anything. + let sealed = gitlawb_core::scan_token::open_scan_token( + &key, + &cid, + &token, + chrono::Utc::now().timestamp(), + ) + .expect("the node's own key opens its own token"); + assert!( + markers + .iter() + .any(|(id, _, created)| *id == sealed.id && *created == sealed.created_at_key), + "the row at the truncation boundary must be one of the poisoned withheld \ + repos; sealed {sealed:?}" + ); + + // A different key must not open it: the seal, not an encoding, is what withholds. + let other = gitlawb_core::scan_token::new_key(); + assert!( + gitlawb_core::scan_token::open_scan_token( + &other, + &cid, + &token, + chrono::Utc::now().timestamp() + ) + .is_none(), + "a token that opens under any key but the node's own is not sealed" + ); + + // Token LENGTH must not vary with the sealed row. + let now = chrono::Utc::now().timestamp(); + let short = gitlawb_core::scan_token::seal_scan_token( + &key, + &cid, + &gitlawb_core::scan_token::ScanPosition { + created_at_key: "2020-01-01T00:00:00+00:00".into(), + id: "a/b".into(), + }, + now + 60, + ) + .unwrap(); + let long = gitlawb_core::scan_token::seal_scan_token( + &key, + &cid, + &gitlawb_core::scan_token::ScanPosition { + created_at_key: "2020-01-01T00:00:00+00:00".into(), + id: format!("did:key:z6MkAVeryLongOwnerKeyIdentifier/{}", "n".repeat(48)), + }, + now + 60, + ) + .unwrap(); + assert_eq!( + short.len(), + long.len(), + "tokens sealing rows of very different id lengths must be byte-identical in \ + length, or the length is a side channel for the withheld row — which the \ + substring assertions above structurally cannot see" + ); + } + + /// Scenario 10: tampered, foreign-CID, and expired tokens are ABSENT, uniformly. + /// + /// Each of the three failure classes must produce exactly the front-started response + /// a tokenless request gets: same status, same body shape, and — the decisive part — + /// an emitted continuation sealing the FRONT window's last row, not the row the + /// rejected token named. Never an error, never a resumed position, and no way to + /// tell the three classes apart. + /// + /// The "front-started" half is asserted by opening the EMITTED token and checking + /// which row it seals, which looks over-elaborate until you try the obvious thing. + /// `scan_rows()` cannot separate the two states: a front start reads rows 1-2 and a + /// resume from the rejected position reads rows 3-4, so the counter says 2 either + /// way. The sealed position is the only thing that differs, and without checking it + /// the foreign-CID leg passes under mutation F. + /// + /// No pre-fix RED, for the same reason as the guard above: the probes are minted + /// with `seal_scan_token`, which does not exist pre-fix. Mutation F is its RED. + /// + /// MUTATION F (RED): drop the CID from the associated data and the foreign-CID leg's + /// token is honoured, so the scan resumes at the foreign position. + #[sqlx::test] + async fn scan_token_invalid_variants_start_at_front(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 2; + seed_root_denying_repos(&state, "front", 6, 0).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let key = state.ipfs_scan_token_key.clone(); + let now = chrono::Utc::now().timestamp(); + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.151:5000".parse().unwrap(); + + // Baseline: a tokenless request reads the front window and seals row 2. + let (base_status, base_body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + let base_token = continuation_of(&base_body).expect("baseline emits a token"); + let front = gitlawb_core::scan_token::open_scan_token(&key, &cid, &base_token, now) + .expect("baseline token opens"); + assert_eq!( + front.id, "front-0001", + "fixture precondition: the front window ends at the second seeded row" + ); + + // Probe 1: a byte-flipped token. + let mut bytes: Vec = base_token.bytes().collect(); + let last = bytes.len() - 1; + bytes[last] = if bytes[last] == b'A' { b'B' } else { b'A' }; + let tampered = String::from_utf8(bytes).unwrap(); + + // Probe 2: a well-formed token minted for a DIFFERENT CID, at a position deep + // in the table so honouring it would be unmistakable. + let elsewhere = cid_for_oid(&"f4".repeat(32)); + let foreign = gitlawb_core::scan_token::seal_scan_token( + &key, + &elsewhere, + &gitlawb_core::scan_token::ScanPosition { + created_at_key: scan_order_stamp(3).to_rfc3339(), + id: "front-0003".into(), + }, + now + 3600, + ) + .unwrap(); + + // Probe 3: a token for this CID whose expiry is already past. + let expired = gitlawb_core::scan_token::seal_scan_token( + &key, + &cid, + &gitlawb_core::scan_token::ScanPosition { + created_at_key: scan_order_stamp(3).to_rfc3339(), + id: "front-0003".into(), + }, + now - 1, + ) + .unwrap(); + + for (what, probe) in [ + ("tampered", tampered), + ("foreign-CID", foreign), + ("expired", expired), + ] { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), Some(&probe))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, base_status, + "a {what} token must answer exactly as a tokenless request does, never \ + an error and never a distinguishable status: {body}" + ); + assert_eq!( + body["error"], base_body["error"], + "a {what} token must not change the body shape: {body}" + ); + assert_eq!( + body["message"], base_body["message"], + "a {what} token must not change the message: {body}" + ); + let token = continuation_of(&body) + .unwrap_or_else(|| panic!("the {what} probe answers like a front start: {body}")); + let pos = gitlawb_core::scan_token::open_scan_token(&key, &cid, &token, now) + .expect("the emitted token opens"); + assert_eq!( + pos.id, front.id, + "a {what} token must be treated as ABSENT and the scan must start at the \ + FRONT; resuming from it would honour a position the caller was never \ + handed for this CID" + ); + } + } + + /// Scenario 11: every seal draws a FRESH nonce. + /// + /// This is the property the whole confidentiality claim rests on and the one the + /// other two token guards cannot see: both of them pass unchanged under a constant + /// nonce. Under a stream cipher a repeated nonce repeats the keystream, so two + /// tokens sealed under one nonce XOR to the difference of their plaintexts — and an + /// attacker who can force the node to seal a position they know then recovers a + /// withheld row's fields in full. That is strictly worse than the base64 defect + /// INV-13 records. + /// + /// No pre-fix RED, like the two guards above: it seals through an API that does not + /// exist on the pre-fix head. Mutation H is its RED, and H reddens nothing else, + /// which is the same fact stated from the other side. + /// + /// MUTATION H (RED): fix the nonce to a constant and the two tokens are identical. + #[test] + fn scan_token_seals_are_nonce_fresh() { + let key = gitlawb_core::scan_token::new_key(); + let pos = gitlawb_core::scan_token::ScanPosition { + created_at_key: "2020-01-01T00:00:07+00:00".into(), + id: "did:key:z6MkHiddenOwner/withheld-repo".into(), + }; + let expires = chrono::Utc::now().timestamp() + 3600; + let first = + gitlawb_core::scan_token::seal_scan_token(&key, "bafkcid", &pos, expires).unwrap(); + let second = + gitlawb_core::scan_token::seal_scan_token(&key, "bafkcid", &pos, expires).unwrap(); + + assert_ne!( + first, second, + "sealing the same position twice must produce different bytes — identical \ + tokens mean a reused nonce, and a reused nonce under a stream cipher leaks \ + the withheld plaintext to anyone holding two tokens" + ); + let now = chrono::Utc::now().timestamp(); + for token in [&first, &second] { + let opened = gitlawb_core::scan_token::open_scan_token(&key, "bafkcid", token, now) + .expect("both tokens must still open"); + assert_eq!( + opened, pos, + "nonce freshness must not cost correctness: both seals open to the same \ + position" + ); + } + } + + /// Scenario 12: the degenerate ZERO-ROW resume never 404s. + /// + /// A row count that is an EXACT multiple of the ceiling is the shape whose last + /// emitted token points AT the final row, so the next resume fetches an empty page. + /// The wrap taint is evaluated on `pager.exhausted`, not at any particular break + /// site, so this is covered by construction: an implementation that keys the taint + /// on having fetched a page passes every other scenario here and converts an + /// incomplete search into a false 404 exactly here. + #[sqlx::test] + async fn scan_token_at_table_end_wraps_not_404(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 2; + // 4 rows at ceiling 2: the second rung's token points at the last row. + seed_root_denying_repos(&state, "endstop", 4, 0).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.149:5000".parse().unwrap(); + + let mut token: Option = None; + let mut last = None; + for step in 1..=4 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "no rung of this ladder may 404 — least of all the zero-row one \ + (step {step}): {body}" + ); + match continuation_of(&body) { + Some(next) => token = Some(next), + None => { + last = Some(body); + break; + } + } + } + let last = last.expect("the ladder must terminate at the table end"); + assert_eq!(last["error"], "search_incomplete", "{last}"); + assert!( + last["message"] + .as_str() + .unwrap_or_default() + .contains("scan-wrapped"), + "a resume landing on or past the last row fetches an empty page, which sets \ + `exhausted` and must taint scan-wrapped: {last}" + ); + assert!( + continuation_of(&last).is_none(), + "a wrapped scan emits no token: {last}" + ); + } + /// F3 budget expiry mid-loop: one absolute request budget /// (`ipfs_request_budget_secs`) bounds the whole admitted scan; per-repo /// stages may not each draw a fresh timeout past it. Budget 1s, per-iteration diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 509c60fe..30a20a10 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -521,6 +521,9 @@ mod tests { ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, + ipfs_max_legacy_scan_rows: crate::api::ipfs::MAX_LEGACY_SCAN_ROWS_PER_REQUEST, + ipfs_max_legacy_scan_rules: crate::api::ipfs::MAX_LEGACY_SCAN_RULES_PER_REQUEST, + ipfs_scan_token_key: Arc::new(crate::state::AppState::new_scan_token_key()), ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 30fae910..08dbeb5f 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -480,6 +480,37 @@ pub struct Config { )] pub ipfs_max_legacy_probes: usize, + /// Per-request ceiling on how many repo ROWS the `/ipfs/{cid}` resolver's legacy + /// scan may fetch from the database. The probe ceiling above only starts counting + /// once a probe runs, and the two denial classes that dominate a hostile inventory + /// (quarantine, and a root-scope visibility deny) return before a probe or a visit + /// is spent, so without this an all-denying node paged its ENTIRE repo table for one + /// anonymous request while holding a scarce walk permit. + /// + /// Reach bound: a holder buried past the ceiling is servable in + /// `ceil(repos / ceiling) + 1` token-echoing retries. A truncated scan sheds a + /// retryable 503 carrying a sealed continuation token; the caller echoes it as + /// `?scan=` and the scan resumes where it stopped. No server-side scan state. + /// + /// Floor coupling: raising this knob raises every caller's per-window `/ipfs` work + /// allowance whenever the route limit sits below the derived floor, because the + /// floor must fit one full deep scan's page toll (see `AppState::ipfs_work_budget`). + /// + /// Tuning DOWN trade: token presence is a coarse inventory-size oracle. A ceiling + /// truncation emits a token and a wrapped scan does not, so laddering to the + /// `scan-wrapped` taint tells an anonymous caller the node's total repo count, + /// private and quarantined included, to within one ceiling. Tolled and coarse at + /// the 2048 default; it sharpens as the ceiling is lowered. + /// + /// Must be between 1 and 1_048_576. Default: 2048. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS", + default_value_t = crate::api::ipfs::MAX_LEGACY_SCAN_ROWS_PER_REQUEST, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_legacy_scan_rows: usize, + /// Upper bound on the number of EXPENSIVE visibility walks /// (`allowed_blob_set_for_caller_bounded`, a full-history git walk in a /// blocking thread) a single `/ipfs/{cid}` request may run. Only a blob in a @@ -970,20 +1001,23 @@ mod tests { "the work budget must clear one full legacy search per window" ); - // Tight route limit (1): the floor lifts the work budget to the probe budget - // (256), NOT down to 1 — a single deep search still completes its full scan. + // Tight route limit (1): the floor lifts the work budget to a full deep scan — + // the 256-probe budget PLUS the page toll a 2048-row ceiling costs at 128 rows + // per page (16) = 272, NOT down to 1. A single deep search still completes its + // full scan without self-throttling on either charge. let tight = Config::parse_from(["gitlawb-node", "--ipfs-rate-limit", "1"]); assert_eq!( AppState::ipfs_work_budget(&tight), - 256, - "a tight route limit is floored at the 256-probe budget, not clamped to 1" + 272, + "a tight route limit is floored at probes + pages (256 + 16), not clamped to 1" ); // Raised probe budget lifts the floor with it (the work budget tracks the // effective probe budget, not the constant). The walk cap is set to a DIFFERENT // value in the same config on purpose: the two were one field before the split, // so a floor that silently read the walk cap would return 7 here and still look - // plausible. Only the legacy-probe knob may drive this budget. + // plausible. Only the legacy-probe and legacy-scan-rows knobs may drive this + // budget. let raised = Config::parse_from([ "gitlawb-node", "--ipfs-rate-limit", @@ -995,8 +1029,28 @@ mod tests { ]); assert_eq!( AppState::ipfs_work_budget(&raised), - 1000, - "the floor tracks the operator-raised legacy-probe budget, not the walk cap" + 1016, + "the floor tracks the operator-raised legacy-probe budget (1000) plus the \ + default row ceiling's page toll (16), not the walk cap" + ); + + // The scan-rows knob is coupled to the floor too, and this EXECUTES the coupling + // rather than describing it: every page the ceiling permits is charged to the + // caller's work bucket, so a raised ceiling that did not lift the floor would + // 429 an honest caller part-way down their own token ladder. 4096 rows at 128 + // rows per page is 32 pages, so the floor is 256 + 32. + let wide_scan = Config::parse_from([ + "gitlawb-node", + "--ipfs-rate-limit", + "10", + "--ipfs-max-legacy-scan-rows", + "4096", + ]); + assert_eq!( + AppState::ipfs_work_budget(&wide_scan), + 288, + "raising the row ceiling must raise the work floor by the pages it buys \ + (256 probes + 4096/128 = 32 pages), or a full deep scan self-throttles" ); // 0 route limit disables the derived bucket too (a 0-capacity limiter admits all). diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index 874123c5..474408e5 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -50,8 +50,15 @@ pub enum AppError { #[error("incomplete: {0}")] Incomplete(String), - #[error("search incomplete: {0}")] - SearchIncomplete(String), + /// A bounded search that could not complete. `continuation`, when present, is the + /// sealed scan position the caller echoes as `?scan=` to resume where the search + /// stopped (#173 round 13, F2). It is AEAD-sealed at the mint site, never plaintext: + /// the row it names is by construction one the caller was denied (INV-13). + #[error("search incomplete: {message}")] + SearchIncomplete { + message: String, + continuation: Option, + }, #[error("git error: {0}")] Git(String), @@ -168,10 +175,10 @@ impl IntoResponse for AppError { // legacy-probe or walk ceiling), distinct from the 404 that asserts a // definitive not-found: absence was NOT proven, so the caller should // retry rather than treat it as gone (#173, F2). 503, retryable. - AppError::SearchIncomplete(msg) => ( + AppError::SearchIncomplete { message, .. } => ( StatusCode::SERVICE_UNAVAILABLE, "search_incomplete", - msg.clone(), + message.clone(), ), AppError::Git(msg) => (StatusCode::INTERNAL_SERVER_ERROR, "git_error", msg.clone()), // 504, distinct from the 500 git_error and from the read-gate's 404 / @@ -213,19 +220,31 @@ impl IntoResponse for AppError { } }; - let body = Json(json!({ + let mut body = json!({ "error": code, "message": message, - })); + }); + // A truncated CID search may carry the sealed position the caller echoes as + // `?scan=` to resume. Rendered as a third body field, present only when the + // shed actually left something to resume: a wrapped scan and a throttled + // request both omit it, and its ABSENCE is what tells a caller the ladder is + // over. It is opaque ciphertext; see `gitlawb_core::scan_token`. + if let AppError::SearchIncomplete { + continuation: Some(token), + .. + } = &self + { + body["continuation"] = json!(token); + } - let mut resp = (status, body).into_response(); + let mut resp = (status, Json(body)).into_response(); // Both retryable 503s advertise when to retry: Overloaded (capacity shed) and // SearchIncomplete (a bounded CID search cut short by a cap — retry may complete // it). They ride the shared tail above for body/status, so the header is attached // here rather than in bespoke early returns, keeping each variant handled once. if matches!( self, - AppError::Overloaded(_) | AppError::SearchIncomplete(_) + AppError::Overloaded(_) | AppError::SearchIncomplete { .. } ) { resp.headers_mut().insert( axum::http::header::RETRY_AFTER, diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 4acbb9bc..81db3e99 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -430,6 +430,11 @@ async fn main() -> Result<()> { // request). Default 256 preserves the shipped behaviour. ipfs_max_legacy_probes: AppState::ipfs_legacy_probe_budget(&config), ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, + // Operator-tunable via GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS, read through the same + // helper shape as the probe budget so the knob cannot be a silent no-op. + ipfs_max_legacy_scan_rows: AppState::ipfs_legacy_scan_row_budget(&config), + ipfs_max_legacy_scan_rules: crate::api::ipfs::MAX_LEGACY_SCAN_RULES_PER_REQUEST, + ipfs_scan_token_key: Arc::new(AppState::new_scan_token_key()), ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust, sync_trigger_rate_limiter, diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 2bd6eccb..7fbc6613 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -106,6 +106,33 @@ pub struct AppState { /// before spending a single probe (#173, INV-10). A field for the same test-seam /// reason as the sibling caps. pub ipfs_legacy_scan_page_rows: usize, + /// Per-request ceiling on how many repo ROWS the CID resolver's legacy scan may + /// fetch (default `api::ipfs::MAX_LEGACY_SCAN_ROWS_PER_REQUEST`, operator-tunable via + /// `GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS`). `ipfs_max_legacy_probes` bounds the PROBE + /// fan-out but only starts counting once a probe runs, and quarantine plus a + /// root-scope visibility deny both return before a probe or a visit is spent, so an + /// all-denying inventory paged the whole repo table at zero probes (#173 round 13, F2). + /// Truncating here sheds a retryable 503 carrying a sealed continuation token. + pub ipfs_max_legacy_scan_rows: usize, + /// Per-request ceiling on how many visibility RULES the CID resolver's legacy scan + /// may retain (default `api::ipfs::MAX_LEGACY_SCAN_RULES_PER_REQUEST`). The row + /// ceiling above bounds the row count but not the memory each row drags in: the + /// pager keeps every fetched page's rules for the whole request. Deliberately NOT an + /// operator knob (it is a memory guard, not a reach tradeoff); a field only for the + /// same test-seam reason as the sibling caps. + pub ipfs_max_legacy_scan_rules: usize, + /// Per-boot key sealing the legacy scan's continuation tokens (INV-13). + /// + /// The token is minted from a FETCHED row on a scan that served nothing, so by + /// construction that row is a private or quarantined repo the caller may not read: + /// its `created_at` and its `id` (which carries the owner's DID) are withheld + /// fields. The token is therefore AEAD-SEALED, never signed plaintext and never + /// base64-of-plaintext — integrity is not confidentiality. Random per boot rather + /// than derived or persisted: a scan continuation has no cross-restart meaning (a + /// stale token simply fails to open and the caller restarts at the front, which is + /// the same uniform absent behaviour a tampered token gets), and a per-boot key + /// bounds the window in which any single key seals anything. + pub ipfs_scan_token_key: Arc<[u8; 32]>, /// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` will buffer and /// serve (default `api::ipfs::MAX_SERVED_OBJECT_BYTES`). The serve reads via a /// blocking `git cat-file` and buffers the whole object; without a bound a large @@ -322,6 +349,19 @@ impl AppState { config.ipfs_max_legacy_probes as u32 } + /// Legacy-scan ROW budget wired from the `GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS` knob, + /// the same helper shape as the probe budget above so the knob cannot become a + /// silent no-op if a struct literal drifts back to the bare constant. + pub(crate) fn ipfs_legacy_scan_row_budget(config: &crate::config::Config) -> usize { + config.ipfs_max_legacy_scan_rows + } + + /// A fresh random key for sealing legacy-scan continuation tokens (INV-13). + /// Drawn from the OS CSPRNG at construction; see `ipfs_scan_token_key`. + pub(crate) fn new_scan_token_key() -> [u8; 32] { + gitlawb_core::scan_token::new_key() + } + /// Work-budget capacity for [`ipfs_work_rate_limiter`](Self#structfield.ipfs_work_rate_limiter) /// (R6, KTD6), DERIVED from the route limit rather than a new operator knob. The route /// limiter (`ipfs_rate_limiter`) charges once per request; this separate bucket absorbs @@ -336,11 +376,25 @@ impl AppState { /// The floor is the LEGACY-PROBE knob, not `ipfs_max_repos_walked`. Those were one /// field before the walk cap and the probe budget were split apart, and reading the /// walk cap here would silently size this bucket at 64 instead of 256. + /// + /// The floor also carries the scan's PAGE toll (#173 round 13, F2): every page the + /// legacy scan buys is charged to this same bucket, so a deep scan spends + /// `ceil(ipfs_max_legacy_scan_rows / LEGACY_SCAN_PAGE_ROWS)` tokens on pages on top + /// of its probes. Leaving those out would 429 an honest caller part-way down their + /// own continuation-token ladder, which is the F6 admit-then-429 shape in a new + /// place. The page term uses the CONSTANT page size, not `AppState`'s field: the + /// field is a test seam that shrinks pages to make paging observable, and sizing a + /// production floor from it would inflate the budget by whatever a test chose. pub(crate) fn ipfs_work_budget(config: &crate::config::Config) -> usize { if config.ipfs_rate_limit == 0 { return 0; } - config.ipfs_rate_limit.max(config.ipfs_max_legacy_probes) + let pages = config + .ipfs_max_legacy_scan_rows + .div_ceil(crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS); + config + .ipfs_rate_limit + .max(config.ipfs_max_legacy_probes + pages) } } diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 58a8da43..6770a02a 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -83,6 +83,9 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, + ipfs_max_legacy_scan_rows: crate::api::ipfs::MAX_LEGACY_SCAN_ROWS_PER_REQUEST, + ipfs_max_legacy_scan_rules: crate::api::ipfs::MAX_LEGACY_SCAN_RULES_PER_REQUEST, + ipfs_scan_token_key: Arc::new(crate::state::AppState::new_scan_token_key()), ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), @@ -10810,8 +10813,14 @@ mod tests { let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; + // Budget = one full scan of the single seeded repo: 1 page + 1 probe. The page + // is charged because the legacy scan's DB-facing pages draw on this same bucket + // (#173 round 13, F2) — without that charge a denial-only inventory could be + // re-paged for free by re-requesting. Production never sees a bucket this small: + // `AppState::ipfs_work_budget` floors it at probes + pages, so only a fixture + // that sets the limiter by hand has to do the arithmetic itself. state.ipfs_work_rate_limiter = - crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let fx = seed_cid_repos(&slug, &short, &["fanout"]); @@ -10869,10 +10878,15 @@ mod tests { let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; - // Budget = one full scan of the four seeded repos. A repeat scan from the same - // IP then finds it spent. Keyed on XFF so `oneshot` can choose the source IP. + // Budget = one full scan of the four seeded repos: 1 page + 4 probes. A repeat + // scan from the same IP then finds it spent. Keyed on XFF so `oneshot` can + // choose the source IP. The page term is there because the scan's DB-facing + // pages draw on this same bucket (#173 round 13, F2), so re-requesting cannot + // buy the inventory again for free; all four repos fit in one 128-row page, so + // one page covers the whole scan. Production is floored at probes + pages by + // `AppState::ipfs_work_budget` — only a hand-set limiter does this arithmetic. state.ipfs_work_rate_limiter = - crate::rate_limit::RateLimiter::new(4, Duration::from_secs(3600)); + crate::rate_limit::RateLimiter::new(5, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let names = ["a0", "a1", "a2", "a3"]; @@ -12667,12 +12681,15 @@ mod tests { let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; - // The scan probes both seeded repos (walklimit + walkpublic) per request, so - // size the per-IP budget to admit exactly one full scan (2 probes). A repeat - // scan from the same IP then finds the bucket spent. Keyed on the rightmost - // X-Forwarded-For hop so the test can choose a source IP under `oneshot`. + // The scan reads both seeded repos (walklimit + walkpublic) in one page and + // probes each, so size the per-IP budget to admit exactly one full scan: + // 1 page + 2 probes. A repeat scan from the same IP then finds the bucket spent. + // Keyed on the rightmost X-Forwarded-For hop so the test can choose a source IP + // under `oneshot`. The page is charged because the scan's DB-facing pages draw + // on this same bucket (#173 round 13, F2). Production is floored at + // probes + pages by `AppState::ipfs_work_budget`; a hand-set limiter is not. state.ipfs_work_rate_limiter = - crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + crate::rate_limit::RateLimiter::new(3, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let fx = seed_cid_repos(&slug, &short, &["walklimit"]); @@ -12875,11 +12892,14 @@ mod tests { let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; - // Budget = one full two-repo scan (2 probes), keyed on the rightmost XFF hop - // so `oneshot` can choose a source IP (no socket peer). A repeat scan from the - // same IP then finds the budget spent. + // Budget = one full two-repo scan: 1 page + 2 probes. Keyed on the rightmost XFF + // hop so `oneshot` can choose a source IP (no socket peer). A repeat scan from + // the same IP then finds the budget spent. The page is charged because the + // scan's DB-facing pages draw on this same bucket (#173 round 13, F2); both + // repos fit in one 128-row page. Production is floored at probes + pages by + // `AppState::ipfs_work_budget`, so only a hand-set limiter counts this out. state.ipfs_work_rate_limiter = - crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + crate::rate_limit::RateLimiter::new(3, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; // Identical secret-blob content in both bare clones → one CID resolves to From ae96ffa46706a6d088a412aa02dbf2f715ceaa37 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:27:23 -0500 Subject: [PATCH 52/77] fix(node): budget provenance and fallback walks separately MAX_HISTORY_WALKS_PER_REQUEST is MAX_PIN_SOURCES + 1, which is exactly enough for the first pinner plus every recorded additional source. Those seventeen sources each spend a walk reaching their deny when they are root-readable but path-scoped, so the walk budget is gone by the time the at-cap fallback reaches a public source that record_pin_source dropped at its sixteen-row cap. If that source needs a walk too it is skipped at the ceiling and the request returns 503 search_incomplete on every retry: a public object nobody can ever fetch. The counter is now per phase. gate_and_serve already knows which phase it is in, so provenance and fallback each get their own budget against the same cap, and the fallback's capacity no longer depends on what the provenance phase spent. Total walk work per request stays bounded at twice the cap, a constant, so the amplification story holds. Raising the shared ceiling was the obvious alternative and does not work: the adversary picks how many provenance slots exist, since they are grindable repo ids filling pin_repo_sources, so cap + N denials recreate the exhaustion for any constant N. The test uses real repos with real path-scoped rules that each spend a real walk, proven by the walk log. The existing buried-source test seeds repos that do not exist, which never enter the walk block at all, and that is why it could not see this. The must-not assertion needed changing to mean anything. A CID is content-addressed, so every holder of it has byte-identical content and comparing the served bytes against the public repo's object cannot show which repo served. The gate is pinned instead by a second object held only by a path-denying repo, with the fallback armed for it, asserted to 404 for an anonymous caller. Verified by mutation: re-merging the counters restores the deterministic 503; capping the fallback at what the provenance phase left over is a reserve that still couples them and fails the same way; and dropping the fallback cap entirely unbounds the total. Each reddens exactly one of the two tests, so neither is standing in for the other. --- crates/gitlawb-node/src/api/ipfs.rs | 363 +++++++++++++++++++++++++++- 1 file changed, 356 insertions(+), 7 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f1c05b69..409d3fb0 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -44,8 +44,8 @@ use crate::visibility::{visibility_check, Decision}; /// within ONE request the object can exist under path-scoped rules in many repos, and /// each distinct repo pays its own `spawn_blocking` walk (the memo only dedups the same /// repo). Without a ceiling a single request fans out to O(repos) walks — an -/// amplification sink (INV-10). Once this many walks have run, no further walk is -/// spawned for the rest of the request: any remaining candidate that still needs +/// amplification sink (INV-10). Once this many walks have run IN A PHASE, no further +/// walk is spawned for that phase: any remaining candidate there that still needs /// a walk is skipped (and, with nothing else readable, the request falls through /// to the opaque 404). The bound is deliberately generous: a legitimate caller /// serves on the first repo that grants them, so reaching it requires being @@ -59,6 +59,15 @@ use crate::visibility::{visibility_check, Decision}; /// served, not falsely 503'd as a truncated search. The legacy scan's fan-out is /// separately bounded by `MAX_LEGACY_PROBES_PER_REQUEST`, so widening this by one /// does not loosen that path. +/// +/// The ceiling is charged PER PHASE (#173 round 13, F3), and that is what makes the +/// paragraph above hold for the fallback too: the legacy-scan fallback gets its own +/// equal budget rather than the provenance phase's remainder, so one request can spawn +/// up to `2 * walk_cap` walks in total and no more. Without the split, a source set of +/// root-readable but path-scoped denials spends the whole ceiling reaching its denials, +/// and the fallback armed to find the PUBLIC source `record_pin_source` silently +/// dropped cannot walk to it, a deterministic 503 on every retry for an object that is +/// public. pub(crate) const MAX_HISTORY_WALKS_PER_REQUEST: u32 = crate::db::MAX_PIN_SOURCES as u32 + 1; /// Hard per-request ceiling on how many legacy (NULL-provenance) repositories @@ -496,7 +505,8 @@ pub async fn get_by_cid( // and the legacy scan so both honor the same fan-out ceiling, per-repo memo, and // IP brake. The caller is constant for one request, so `repo.id` alone keys the memo. let mut walk = WalkState { - walks: 0, + provenance_walks: 0, + scan_walks: 0, probes: 0, visits: 0, truncated_by: Vec::new(), @@ -1011,7 +1021,14 @@ struct ResolveCtx<'a> { /// Per-request walk budget + memos, shared across the provenance path and the legacy /// scan so the fan-out ceiling and per-repo memoization span the whole request. struct WalkState { - walks: u32, + /// Walks spent by the PROVENANCE phase, checked against `walk_cap` on its own. + provenance_walks: u32, + /// Walks spent by the legacy-scan fallback, checked against the SAME `walk_cap` + /// but from its own zero. The two phases are budgeted separately because they are + /// not alternatives: the fallback exists precisely to reach a source the + /// provenance set dropped, and a shared counter let the provenance phase's denials + /// spend the budget the fallback needs to get there (#173 round 13, F3). + scan_walks: u32, /// Count of legacy (NULL-provenance) repos actually probed this request, so the /// scan can stop at `ipfs_max_legacy_probes` instead of fanning out to O(repos) /// `acquire` + `cat-file` (#173, F1, INV-10). Only the legacy path bumps it. @@ -1302,17 +1319,41 @@ async fn gate_and_serve( } // Per-request fan-out ceiling (INV-10): once this many walks have run, skip // THIS walk-requiring candidate and keep scanning (a later walk-free copy - // must still serve). `walks` is bumped only inside this block, so walk-free + // must still serve). Only this block bumps a counter, so walk-free // candidates never consume budget. // Both parents bound this loop, under different knobs: #173's // `ipfs_max_history_walks` (an AppState field, seeded from config) and // #174's `GITLAWB_IPFS_MAX_REPOS_WALKED`. Honor the tighter of the two, so // neither knob silently stops working after the merge. + // + // The cap is charged PER PHASE (#173 round 13, F3): the provenance path and + // the legacy-scan fallback each get their own `walk_cap`, so the total walk + // work one request can buy is `2 * walk_cap` and no more. A single shared + // counter made a public object permanently unservable: every provenance + // source that is root-readable but path-scoped needs a walk to reach its + // deny, so a full source set of them spends the whole ceiling, and the + // fallback armed to find the source `record_pin_source` dropped then has + // nothing left to walk with: it skips that source here, taints, and every + // retry reproduces the same 503. + // + // Raising a single shared ceiling instead was rejected. The adversary + // controls how many provenance slots exist (they are grindable repo ids + // filling `pin_repo_sources`), so for any constant N a set of + // `walk_cap + N` path-scoped denials re-creates the exhaustion. Only a + // budget the provenance phase cannot draw from bounds the fallback's reach + // independently of what the source set contains. The taint name stays + // "walk-cap": to an operator the meaning is unchanged (a walk ceiling cut + // the search), and the knobs still mean what they say, now per phase. let walk_cap = std::cmp::min( state.ipfs_max_history_walks as usize, state.config.ipfs_max_repos_walked, ); - if walk.walks as usize >= walk_cap { + let spent = if legacy_scan { + walk.scan_walks + } else { + walk.provenance_walks + }; + if spent as usize >= walk_cap { // The walk ceiling truncated the search: a later repo (possibly one that // authorizes this caller) is left unwalked, so absence is unproven — // record it so the tail returns 503, not a false 404 (#173, F2). @@ -1341,7 +1382,11 @@ async fn gate_and_serve( } } } - walk.walks += 1; + if legacy_scan { + walk.scan_walks += 1; + } else { + walk.provenance_walks += 1; + } let rp = repo_path.clone(); let r = rules.to_vec(); @@ -2196,6 +2241,43 @@ mod tests { .to_string() } + /// Walk-counting shim that runs the REAL walk (`state.git_bin`): each `rev-list` + /// appends one line to `log`, then every invocation execs the real `git`, so the + /// allowed-set a walk produces is the repo's genuine one. + /// + /// `walk_logging_fake_git` below answers every subcommand with nothing, so under it + /// EVERY walked repo yields an empty allowed set and no repo can ever authorize. The + /// per-phase budget tests need one candidate to deny after a real walk and a later + /// one to allow after another, so they need the real sets and the tally both. + #[cfg(unix)] + fn walk_logging_real_git(dir: &std::path::Path, log: &std::path::Path) -> String { + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-list) echo walk >> \"{}\" ;;\n\ + esac\n\ + exec git \"$@\"\n", + log.display() + ); + let git_path = dir.join("walkgit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + git_path.to_str().unwrap().to_string() + } + + /// How many expensive walks the shim above has recorded so far. + #[cfg(unix)] + fn walks_logged(log: &std::path::Path) -> usize { + std::fs::read_to_string(log) + .map(|s| s.lines().count()) + .unwrap_or(0) + } + /// Fake git for the WALK only (`state.git_bin`): empty refs, `rev-parse` /// resolves, and each `rev-list` appends one line to `log` and prints nothing — /// every walked repo yields an EMPTY allowed-set (path-gate deny verdict) and @@ -2532,6 +2614,273 @@ mod tests { ); } + /// A reader DID that is on no rule in these fixtures, so every path-scoped rule + /// naming it denies the anonymous caller at the rule's path. + #[cfg(unix)] + const OTHER_READER: &str = "did:key:z6MkU3IpfsReaderCCCCCCCCCCCCCCCCCCCCCCCC"; + + /// Seed a repo holding `content` at `/src/secret.txt` and give it a path-scoped + /// rule over `/src/**` naming a reader that is not the caller. The repo stays + /// readable at "/" (the rule does not match "/", so the mirror row's public flag + /// decides), which is what makes the object cost a real allowed-set walk before it + /// is denied: a root deny would short-circuit ahead of the walk and spend nothing. + #[cfg(unix)] + async fn seed_path_denying_repo( + state: &crate::state::AppState, + tmp: &std::path::Path, + owner: &str, + name: &str, + content: &[u8], + ) -> (String, String) { + let (id, oid) = seed_repo_with_blob(state, tmp, owner, name, content).await; + state + .db + .set_visibility_rule( + &id, + "/src/**", + crate::db::VisibilityMode::B, + &[OTHER_READER.to_string()], + owner, + ) + .await + .expect("seed the path-scoped deny rule"); + (id, oid) + } + + /// F3 per-phase walk budgets: a PUBLIC source that only the legacy-scan fallback + /// can reach must still serve after path-scoped provenance denials spent the whole + /// walk cap. + /// + /// One shared `walks` counter made that impossible. Every provenance source that is + /// root-readable but path-scoped needs its own allowed-set walk to reach its deny, + /// so `MAX_PIN_SOURCES + 1` such sources consume the entire cap; the fallback the + /// at-cap/incomplete markers then arm has nothing left to spend, skips its first + /// walk-needing candidate at `walk-cap`, and the request tails to a retryable 503 + /// that every retry reproduces. A public object, permanently unservable. + /// + /// The existing buried-public test cannot see this: its extra repos do not exist on + /// disk, so they never reach the `!already` block and consume no walk. This fixture + /// uses REAL repos with REAL denying rules, and the walk log is what proves each one + /// genuinely spent a walk rather than being skipped for free. + /// + /// Both caps are set to 2, so `walk_cap` is 2 per phase. The first request runs + /// WITHOUT the fallback armed and pins the provenance phase's own bound (exactly 2 + /// walks, never more, for a complete source set). The second arms the fallback and + /// is the RED: pre-fix the public repo is skipped at `walk-cap` and the request 503s. + /// The third pins that the fresh scan budget is capacity, not a gate change: an + /// object held ONLY by a path-denying repo is still not served to the anonymous + /// caller, with the fallback armed for it too. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_fallback_reaches_public_source_past_provenance_walk_spend( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_real_git(tmp.path(), &walk_log); + // `walk_cap` is the min of the two knobs, so both go to 2. + state.ipfs_max_history_walks = 2; + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 2; + state.config = Arc::new(cfg); + + // Identical content in every holder, so one CID resolves to one oid that all of + // them carry. Iteration is `(created_at, id)` ASC, so insert order is scan order. + let content = b"per-phase walk budget proof\n"; + let (prov_one, oid) = + seed_path_denying_repo(&state, tmp.path(), "z6f3phase", "provdeny-one", content).await; + let (prov_two, _) = + seed_path_denying_repo(&state, tmp.path(), "z6f3phase", "provdeny-two", content).await; + // The fallback's holder. Its rule IS path-scoped (so the object still costs a + // walk) but covers a path this object is not at, so the walk's allowed-set + // decides on the mirror row's public flag and ALLOWS. A path-scoped rule can + // never name an anonymous reader, so this is the only shape in which a walked + // repo authorizes anon. + let (public_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f3phase", "pubreach", content).await; + state + .db + .set_visibility_rule( + &public_id, + "/decoy/**", + crate::db::VisibilityMode::B, + &[OTHER_READER.to_string()], + "z6f3phase", + ) + .await + .unwrap(); + // A second object held ONLY by a path-denying repo, for the denial-class check. + let denied_content = b"held only where anon is denied\n"; + let (denied_id, denied_oid) = seed_path_denying_repo( + &state, + tmp.path(), + "z6f3phase", + "deniedsolo", + denied_content, + ) + .await; + + // Provenance: the two denying repos are the recorded sources of `oid`; the + // public holder is NOT, which is exactly the dropped-source case. + state.db.record_pin_source(&oid, &prov_one).await.unwrap(); + state.db.record_pin_source(&oid, &prov_two).await.unwrap(); + state + .db + .record_pin_source(&denied_oid, &denied_id) + .await + .unwrap(); + state + .db + .mark_pin_sources_incomplete(&denied_oid, "") + .await + .unwrap(); + + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + let denied_cid = seed_legacy_pin_for_oid(&state, &denied_oid).await; + let router = ipfs_router(state.clone()); + + // 1. Provenance only: the source set carries no incompleteness signal, so no + // fallback runs. Both sources walk and deny, and the phase spends its cap + // exactly, never more, whatever the fallback later gets. + let (status, body) = + status_and_body(router.clone().oneshot(get_cid(&cid, None)).await.unwrap()).await; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "a complete source set that denies everywhere is a definitive miss: {body}" + ); + assert_eq!( + walks_logged(&walk_log), + 2, + "the provenance phase must spend exactly its own walk_cap of 2: two REAL \ + path-denying sources, each walked to reach its deny" + ); + + // 2. Arm the fallback (the node's own record that a source is missing) and the + // buried public holder must serve, on the scan phase's own budget. + state + .db + .mark_pin_sources_incomplete(&oid, "") + .await + .unwrap(); + std::fs::remove_file(&walk_log).unwrap(); + let resp = router.clone().oneshot(get_cid(&cid, None)).await.unwrap(); + let status = resp.status(); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!( + status, + StatusCode::OK, + "a public source reachable only through the fallback must serve even after \ + path-scoped provenance denials spent the whole walk cap: {}", + String::from_utf8_lossy(&body) + ); + assert_eq!( + &body[..], + content.as_slice(), + "the served bytes must be the public holder's object" + ); + assert_eq!( + walks_logged(&walk_log), + 3, + "two provenance-phase walks plus ONE scan-phase walk: the phases hold \ + separate budgets and neither exceeds the cap of 2" + ); + + // 3. The fresh scan budget is capacity, not a gate change: an object held only + // where anon is denied stays denied, fallback armed and all. + std::fs::remove_file(&walk_log).unwrap(); + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid(&denied_cid, None)) + .await + .unwrap(), + ) + .await; + assert_ne!( + status, + StatusCode::OK, + "a path-scoped deny must still deny under the per-phase budgets: {body}" + ); + assert_eq!( + status, + StatusCode::NOT_FOUND, + "every holder of that object reached a real deny verdict, so the miss is \ + definitive rather than truncated: {body}" + ); + } + + /// F3 must-not: the per-phase split raises the total walk work to `2 * walk_cap` + /// and no further. Two provenance sources spend the provenance budget; three more + /// path-denying repos, none of them recorded sources, offer the fallback more + /// walk-needing candidates than its own budget. The scan takes two and skips the + /// rest at `walk-cap`, so the request tails to the tainted 503 rather than walking + /// on. + /// + /// The walk count is asserted BEFORE the status so an unbounded scan fails HERE, + /// on the bound, and not on some downstream difference. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_per_phase_walk_budgets_stay_bounded_at_twice_the_cap(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_real_git(tmp.path(), &walk_log); + state.ipfs_max_history_walks = 2; + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 2; + state.config = Arc::new(cfg); + + let content = b"bounded total walk work\n"; + let (prov_one, oid) = + seed_path_denying_repo(&state, tmp.path(), "z6f3total", "provdeny-one", content).await; + let (prov_two, _) = + seed_path_denying_repo(&state, tmp.path(), "z6f3total", "provdeny-two", content).await; + for name in ["fallback-one", "fallback-two", "fallback-three"] { + seed_path_denying_repo(&state, tmp.path(), "z6f3total", name, content).await; + } + state.db.record_pin_source(&oid, &prov_one).await.unwrap(); + state.db.record_pin_source(&oid, &prov_two).await.unwrap(); + state + .db + .mark_pin_sources_incomplete(&oid, "") + .await + .unwrap(); + + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid(&cid, None)) + .await + .unwrap(), + ) + .await; + let walks = walks_logged(&walk_log); + assert!( + walks <= 4, + "total walk work must stay within 2 * walk_cap = 4 however many walk-needing \ + candidates the fallback is offered, got {walks}" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the fallback's own budget runs out on the surplus candidates, so the tail \ + is the truncated-search 503: {body}" + ); + assert_eq!(body["error"], "search_incomplete", "{body}"); + } + /// F2 visit ceiling: `ipfs_max_repo_visits` bounds the acquire+probe cost class /// (each visit can be a full Tigris archive fetch on a cache miss). Unlike the /// walk cap there is no cheap way to keep scanning, so exhaustion STOPS the scan From 0d7711c08917a70f982391249439485ddce382b0 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:59:02 -0500 Subject: [PATCH 53/77] fix(node): bound the pre-walk CID resolve on its own short budget get_by_cid takes the global walk permit and the per-source permit before resolving the CID, and that lookup was clamped only by the 600s request budget. A well-formed CID with no pinned row does no probe and no walk, but against a stalled pool it held a scarce walk slot for the full window, so requests from enough distinct sources could reject every real retrieval at admission while nothing was walking. The lookup now runs under its own budget, defaulting to ten seconds and clamped by the request budget, so a larger value degrades rather than extends. Admission still comes first. Resolving before admission was the other way to fix this and it trades one amplification for another: taking the read out from behind the gate lets unadmitted anonymous callers stack concurrent queries, and shedding a flood cheaply before it reaches the database is what admission-first is for. Only that one await moves. Everything after it stays on the request budget because it runs interleaved with work the caller has already been admitted for: the per-oid source lookup runs after real probing from the second candidate on, the marker pair only on a provenance miss, and the pager fetches between pages that cost probes. A short clock anchored at admission would be long spent by then, so widening it would shed requests that are making progress. The region comment now carries that assignment per await. The shed names the knob an operator can actually turn, separately from the request-budget shed, so the two are distinguishable in the body and the log. Verified by mutation: reverting the clamp overruns the short budget, and anchoring one short clock across the whole permit-held region sheds a slow but progressing walk. The second leaves the first test green, which is what makes it a boundary case rather than a second removal. --- .env.example | 9 + README.md | 1 + crates/gitlawb-node/src/api/ipfs.rs | 260 +++++++++++++++++++++++++++- crates/gitlawb-node/src/config.rs | 92 +++++++++- 4 files changed, 355 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index 3126c8c7..41a484a6 100644 --- a/.env.example +++ b/.env.example @@ -253,6 +253,15 @@ GITLAWB_IPFS_MAX_REPO_VISITS=1024 # Must be 1..=3153600000 (100 years): the node derives an Instant deadline from # this value, and a larger one cannot be represented. Default 600. GITLAWB_IPFS_REQUEST_BUDGET_SECS=600 +# Shorter budget (seconds) for the pre-walk CID resolve: the lookup that maps a +# requested CID to its git oid(s), which runs while the scarce walk admission is +# already held. A well-formed CID with no pin row does no probe and no walk work, +# so without this a stalled lookup could hold a walk slot for the whole request +# budget while nothing walked. The effective deadline is the lesser of this and +# the remaining request budget; walk and probe work stay on the request budget, +# so a slow but progressing scan is never shed by it. +# Must be 1..=3153600000 (100 years). Default 10. +GITLAWB_IPFS_RESOLVE_BUDGET_SECS=10 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. GITLAWB_IPFS_RATE_LIMIT=600 diff --git a/README.md b/README.md index cb988551..28827110 100644 --- a/README.md +++ b/README.md @@ -357,6 +357,7 @@ Important node settings: | `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. The effective cap is the tighter of this knob and the node's internal per-request history-walk ceiling of 17 (`MAX_PIN_SOURCES + 1`), so a value above 17 has no effect while a value below it does tighten the cap. Default 64. | | `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate. Also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | | `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting and each also run under their own deadline (the lesser of `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` and the remaining budget), reaped via process-group teardown, so a hung `cat-file` cannot hold the request's walk slot past it. One hang path is still unbounded: the probe's object-store readability check is a plain filesystem sweep with nothing to reap, so a wedged filesystem can hold the slot past the deadline. Default 600. Accepted range is 1 to 3153600000 (100 years), since the node derives a deadline from this value and a larger one cannot be represented. | +| `GITLAWB_IPFS_RESOLVE_BUDGET_SECS` | Shorter budget for the pre-walk CID resolve inside an admitted `/ipfs/{cid}` request: the lookup that maps the requested CID to its git oid(s), which runs while the scarce walk admission is already held. A well-formed CID with no pin row does no probe and no walk work, so without this it could hold a walk slot for the whole request budget while nothing walked, and enough such requests shed every real retrieval at admission. The effective deadline is the lesser of this and the remaining request budget, so a value above `GITLAWB_IPFS_REQUEST_BUDGET_SECS` degrades to the request budget. Only the resolve is on this clock; walk and probe work stay on the request budget, so a slow but progressing scan is never shed by it. Default 10. Accepted range is 1 to 3153600000 (100 years). | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 409d3fb0..95f50b0b 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -447,6 +447,18 @@ pub async fn get_by_cid( _per_source: caller_permit, }); + // A SECOND, much shorter absolute clock, anchored here at admission, bounding only + // the pre-walk CID resolve below (#174 F4). The request budget alone is 600s by + // default, and a syntactically valid CID with no `pinned_cids` row runs zero probes + // and zero walks, so a resolve stalled in Postgres held these scarce permits for the + // whole 600s while nothing walked; enough distinct source keys doing that + // capacity-503 every real `/ipfs` retrieval at admission. Admission deliberately + // stays FIRST: resolving before taking it would let arbitrarily many unadmitted + // permissionless callers stack concurrent DB queries, trading one amplification for + // another, so the repair is a shorter deadline on the stage rather than a reorder. + let resolve_deadline = std::time::Instant::now() + + std::time::Duration::from_secs(state.config.ipfs_resolve_budget_secs); + // Caller DID (owned): the `spawn_blocking` closures below cannot borrow the // handler's `auth` extension, so resolve it once here. let caller_owned = auth.as_ref().map(|e| e.0 .0.as_str().to_string()); @@ -460,6 +472,20 @@ pub async fn get_by_cid( // needed. Defined once here so the clamp sites on the provenance path share one // definition (the legacy-scan preload below keeps its own `budget_shed` inside // its nested scope). + // + // Which of the two clocks each await runs on (#174 F4), enumerated once here: + // - `oids_for_cid` runs on the SHORT resolve budget (clamped by the request + // budget). It is the one await that decides whether the request does any + // admitted work at all; nothing has been paid for yet when it runs, so a shed + // there discards nothing but the permits it is holding. + // - EVERYTHING after it stays on the FULL request budget. `pin_sources_for_oid` + // runs once per oid candidate and from the second candidate on runs after real + // probe and walk work; the marker pair runs only on a provenance miss, which is + // after the per-source loop may already have walked; the per-source trio and the + // legacy pager's fetches interleave with admitted walk work by construction. A + // short deadline anchored at admission would be long spent by the time those run + // in a legitimately slow scan, so putting any of them under it sheds a + // PROGRESSING request rather than an idle one. let budget_shed = || { AppError::Overloaded(format!( "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" @@ -477,7 +503,16 @@ pub async fn get_by_cid( // when the chosen one is withheld or absent while another is readable (#173). // An empty result is an opaque 404, uniform with a genuine not-found and a // visibility denial. - let oids = match tokio::time::timeout(remaining(), state.db.oids_for_cid(&canonical_cid)).await + // + // Clamped to the LESSER of the resolve budget and the request budget, so a resolve + // budget set larger than the request budget degrades to the request budget instead + // of extending it. + let resolve_remaining = resolve_deadline.saturating_duration_since(std::time::Instant::now()); + let oids = match tokio::time::timeout( + std::cmp::min(resolve_remaining, remaining()), + state.db.oids_for_cid(&canonical_cid), + ) + .await { Ok(Ok(v)) => v, // Bare conversion, never `AppError::Internal`: a connection-class sqlx failure @@ -486,6 +521,22 @@ pub async fn get_by_cid( // so a stalled pool and a closed pool stay distinguishable to the caller. Ok(Err(e)) => return Err(e.into()), Err(_elapsed) => { + // Name the clock that actually bound this await, in the log AND in the body: + // the two budgets are separately settable, so pointing an operator at the + // knob that did nothing here is the same defect as not naming one at all. + // Compared as DEADLINES, not as remainders read at two different instants: + // when the two clocks coincide the later read is always the smaller one, so + // a remainder comparison would attribute a tie to whichever was read second. + if resolve_deadline <= request_deadline { + tracing::warn!( + resolve_budget_secs = state.config.ipfs_resolve_budget_secs, + "/ipfs oids_for_cid exceeded the pre-walk resolve budget \ + (GITLAWB_IPFS_RESOLVE_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(AppError::Overloaded(format!( + "ipfs resolve incomplete (resolve budget) for CID {cid_str}; retry shortly" + ))); + } tracing::warn!( budget_secs = state.config.ipfs_request_budget_secs, "/ipfs oids_for_cid exceeded the request budget \ @@ -5933,6 +5984,213 @@ mod tests { ); } + /// F4 (#174 round 13): the pre-walk resolve carries its OWN short budget. The + /// clamp above proves `oids_for_cid` cannot run unbounded, but its deadline is the + /// 600s request budget, and a CID with no `pinned_cids` row does zero probe and + /// zero walk work. Under a stalled pool such a request held the scarce walk slot + /// for that whole window while nothing walked, so requests from enough distinct + /// source keys capacity-503'd every real `/ipfs` retrieval at admission. + /// + /// The request budget is left at its 600s DEFAULT here on purpose: that is the + /// whole point of the scenario, since the short resolve budget, not the long + /// request budget, is what must end this request. + /// + /// Load-bearing: without the resolve clamp the stalled lookup runs to the 600s + /// request budget and blows past the 10s wrap (RED). MUTATION (RED): revert the + /// clamp to `remaining()` only, the pre-fix shape. + #[sqlx::test] + async fn get_by_cid_stalled_resolve_frees_walk_permit_within_resolve_budget( + pool: sqlx::PgPool, + ) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held/freed permit is directly observable; + // per-source cap permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + cfg.ipfs_resolve_budget_secs = 1; + assert_eq!( + cfg.ipfs_request_budget_secs, 600, + "the request budget stays at its default: this scenario proves the SHORT \ + budget is what sheds" + ); + state.config = Arc::new(cfg); + + let sem = state.git_ipfs_walk_semaphore.clone(); + // A well-formed CID with no `pinned_cids` row: an anonymous caller's request + // that will do no admitted work at all once the lookup answers. + let cid = cid_for_oid(&absent_oid()); + let router = ipfs_router(state); + + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE pinned_cids IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.85:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.clone().oneshot(get_cid(&cid, Some(peer))), + ) + .await + .expect( + "the resolve clamp must return within the SHORT budget; on the request budget \ + alone the stalled lookup holds for 600s", + ) + .unwrap(); + let elapsed = started.elapsed(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a resolve blocked past the resolve budget must shed a retryable 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~the resolve budget (1s); got {elapsed:?} \ + (on the request budget alone it runs for 600s)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + // INV-24: the knob an operator can turn must be the one the shed names. The two + // budgets are separately settable, so a body naming the request budget here + // would point at a knob that did nothing. + assert!( + body.contains("resolve budget"), + "the shed must name the RESOLVE budget so it maps to \ + GITLAWB_IPFS_RESOLVE_BUDGET_SECS; got: {body}" + ); + assert!( + !body.contains("request budget"), + "the resolve shed must not name the request budget, which is untouched at \ + 600s here and would send an operator to the wrong knob; got: {body}" + ); + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must be freed on the resolve-budget shed path, not held \ + for the stall" + ); + + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + let resp2 = router.oneshot(get_cid(&cid, Some(peer))).await.unwrap(); + assert_eq!( + resp2.status(), + StatusCode::NOT_FOUND, + "with the permit freed and the lock released, a follow-up is ADMITTED and \ + answers 404 (no pin was seeded), never capacity-503'd" + ); + } + + /// F4 must-not (#174 round 13): the short resolve budget bounds the RESOLVE and + /// nothing else. A request whose resolve answers promptly and then spends real time + /// in an admitted visibility walk is PROGRESSING, and shedding it would convert a + /// slow-but-correct retrieval into a 503 on a box that is merely loaded. + /// + /// The resolve budget is 1s while the walk sleeps ~2s per `rev-list`, so any + /// deadline that reaches past `oids_for_cid` ends this request before it can serve. + /// The shim execs the REAL git after sleeping, so the allowed-set the walk produces + /// is the repo's genuine one and the 200 is a real serve, not an artifact. + /// + /// MUTATION (RED): anchor the region's `remaining()` on the resolve deadline (the + /// plausible over-wide re-implementation: one short admission-anchored clock for + /// the whole permit-held region) and this 200 becomes a budget 503. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_slow_walk_not_shed_by_resolve_budget(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + + let content = b"slow but progressing\n"; + let (repo_id, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6slowwalk", "holder", content).await; + // Path-scoped, so serving costs a real reachability walk: the rule withholds a + // path the seeded blob is NOT under (it lives at `src/secret.txt`), so anon is + // allowed and the request must reach a 200 the slow way. + state + .db + .set_visibility_rule( + &repo_id, + "withheld/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderDDDDDDDDDDDDDDDDDDDDDDDD".to_string()], + "z6slowwalk", + ) + .await + .unwrap(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + // Sleep only on the reachability walk, then exec the real git, so the delay + // lands inside the admitted region and after the resolve has already answered. + let shim = tmp.path().join("slowwalkgit"); + std::fs::write( + &shim, + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-list) sleep 2 ;;\n\ + esac\n\ + exec git \"$@\"\n", + ) + .unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&shim).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&shim, perm).unwrap(); + } + state.git_bin = shim.to_str().unwrap().to_string(); + + let mut cfg = (*state.config).clone(); + cfg.ipfs_resolve_budget_secs = 1; + state.config = Arc::new(cfg); + + let peer: SocketAddr = "203.0.113.86:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + let elapsed = started.elapsed(); + let status = resp.status(); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!( + status, + StatusCode::OK, + "a walk that outlives the SHORT resolve budget must still serve: the resolve \ + budget bounds the pre-walk lookup, never admitted walk work. Got: {}", + String::from_utf8_lossy(&body) + ); + assert_eq!( + &body[..], + content.as_slice(), + "the served bytes must be the seeded object's" + ); + // Anti-vacuity: without a walk that genuinely outlives the 1s resolve budget, + // the 200 above would prove nothing about the boundary. + assert!( + elapsed >= std::time::Duration::from_secs(2), + "the request must actually have spent longer than the 1s resolve budget in \ + the walk; got {elapsed:?}, so the shim's sleep never ran" + ); + } + /// F2 (#174), second lockable site: `pin_sources_for_oid` runs once per candidate /// oid, still inside the admission-held region, and was likewise a bare await. /// diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 08dbeb5f..54841a4c 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -1,13 +1,13 @@ use clap::Parser; use std::path::PathBuf; -/// Upper bound on `git_service_timeout_secs` and `ipfs_request_budget_secs`, in seconds -/// (100 years). +/// Upper bound on `git_service_timeout_secs`, `ipfs_request_budget_secs`, and +/// `ipfs_resolve_budget_secs`, in seconds (100 years). /// -/// Two consumers now, so a future tightening moves both. `ipfs_request_budget_secs` -/// derives only the `Instant` addition in `get_by_cid`, not the lease-steal multiply -/// below, but it shares this ceiling because the defect class and the "set it very large -/// to disable" contract are the same. +/// Three consumers now, so a future tightening moves all of them. `ipfs_request_budget_secs` +/// and `ipfs_resolve_budget_secs` derive only the `Instant` addition in `get_by_cid`, not the +/// lease-steal multiply below, but they share this ceiling because the defect class and the +/// "set it very large to disable" contract are the same. /// /// The knob is not just stored, it is arithmetic input: the write path derives the /// per-repo lease steal bound from it (`* 2 + 60`), and #174 routed it into @@ -600,6 +600,40 @@ pub struct Config { )] pub ipfs_request_budget_secs: u64, + /// Budget for the PRE-WALK CID resolve inside `get_by_cid`, in seconds: the + /// `oids_for_cid` lookup that maps the requested CID to its git oid(s), which runs + /// while the scarce walk admission (the global pool permit plus the per-source + /// sub-permit) is already held. + /// + /// It exists because that one await decides whether the request does any admitted + /// work at all. A syntactically valid CID with no `pinned_cids` row runs zero probes + /// and zero walks, so under a stalled or saturated pool it would otherwise occupy a + /// walk slot for the whole `ipfs_request_budget_secs` window (600s by default) while + /// nothing is walking, and enough distinct source keys doing that reject every real + /// `/ipfs` retrieval at admission. The other repair, resolving the CID before taking + /// admission, was rejected: admission stays FIRST so an anonymous flood sheds before + /// touching the database at all, and moving the read ahead of it would let arbitrarily + /// many unadmitted permissionless callers stack concurrent DB queries. + /// + /// The effective deadline is the lesser of this and the remaining request budget, so a + /// value larger than `ipfs_request_budget_secs` degrades to the request budget rather + /// than extending it. Only the resolve is on this clock; every later stage stays on the + /// full request budget, because from the second oid candidate on those run after real + /// probe and walk work and a short deadline anchored at admission would shed a + /// legitimately slow but progressing scan. + /// + /// Must be positive, and no larger than `GIT_SERVICE_TIMEOUT_SECS_MAX`, for the same + /// representability reason as the request budget above: `get_by_cid` derives the + /// resolve deadline as `Instant::now() + Duration::from_secs(this)`, and that addition + /// panics on overflow in release builds too. Default: 10s. + #[arg( + long, + env = "GITLAWB_IPFS_RESOLVE_BUDGET_SECS", + default_value_t = 10, + value_parser = clap::value_parser!(u64).range(1..=GIT_SERVICE_TIMEOUT_SECS_MAX) + )] + pub ipfs_resolve_budget_secs: u64, + /// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The /// route is publicly reachable (`optional_signature`) and each request can drive /// a full-history git walk, so it carries a per-IP flood brake in addition to the @@ -1138,6 +1172,52 @@ mod tests { ); } + #[test] + fn ipfs_resolve_budget_secs_defaults_to_10_and_rejects_zero() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_resolve_budget_secs, + 10 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-resolve-budget-secs", "3"]) + .ipfs_resolve_budget_secs, + 3 + ); + // 0 would shed every /ipfs request at the pre-walk resolve (unconditional + // 503); clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-resolve-budget-secs", "0"]).is_err() + ); + // The ceiling is shared with the request budget: at the max it parses and the + // derived deadline is still representable, past it clap rejects. + let at_max = Config::try_parse_from([ + "gitlawb-node", + "--ipfs-resolve-budget-secs", + &GIT_SERVICE_TIMEOUT_SECS_MAX.to_string(), + ]) + .expect("the documented maximum must parse"); + assert_eq!( + at_max.ipfs_resolve_budget_secs, + GIT_SERVICE_TIMEOUT_SECS_MAX + ); + assert!(std::time::Instant::now() + .checked_add(std::time::Duration::from_secs( + at_max.ipfs_resolve_budget_secs + )) + .is_some()); + for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] { + assert!( + Config::try_parse_from([ + "gitlawb-node", + "--ipfs-resolve-budget-secs", + &over.to_string(), + ]) + .is_err(), + "{over} is past the representable ceiling and must be rejected at parse time" + ); + } + } + /// #174 (RED-before/GREEN-after): the upper bound is what keeps the deadline derived /// from this knob in range. `get_by_cid` builds the request budget as /// `Instant::now() + Duration::from_secs(this)` (api/ipfs.rs), and that addition is an From 342dd48d2426a8870423cd537781b52d03e8f175 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:12:24 -0500 Subject: [PATCH 54/77] fix(review): mint a continuation at every ceiling and widen the token Code review found two ways the scan ladder still dead-ends, both the same shape the round has been fixing: a truncation that sheds without a token, which this design reads as "the ladder is over", so the caller restarts at the front forever and everything past that point is unreachable. The continuation was wired into the row and rules ceilings but not the probe and visit ceilings, and those two bind first on any inventory with root-readable repos, long before the row ceiling. Every fixture seeded root-denying repos, so no probe was ever charged and no test could reach the arms that were missing it. All four breaks stop at the same cursor, so all four now carry it. The token's field width was 128 bytes against a repo id the write path admits at 356 (255 owner, a slash, 100 name); a plain full DID with a long name already passes 128. A repo over the width at a truncation boundary failed the seal and shed tokenless, and repo names are peer-controlled, so the boundary was steerable rather than unlucky. Fields are now sized separately, 64 for the timestamp key and 384 for the id, and the version byte moves so older tokens fail open to absent instead of mis-parsing. Tokens stay constant length at 668 characters. The sweep no longer exits for good on a failed pass. Making it periodic was the point of the round, and returning on the first error meant one deadlock disabled repair until the next boot, silently. It re-arms on a longer interval, escalates the log once failures are consecutive, and backs off again when a run repairs nothing so a table that will never repair stops costing 64 object reads every five minutes. The rule ceiling counts retained bytes instead of rule rows and is checked where the rules are actually fetched, since a page can carry unboundedly many rules and rule count is the wrong unit for a memory bound. It skips a short final page: without that, a complete scan of an absent object turns into a permanent 503. Two docs contradicted the code and are corrected: the walk cap is charged per phase now, so a request can spend twice the knob, and the README's retry bound holds when the row ceiling binds. Verified by mutation: reverting either ceiling's continuation strands the holder, narrowing the token width fails the seal at the boundary, restoring the terminal return stops the sweep, counting rules instead of bytes lets a page overshoot, and removing the idle backoff keeps a fruitless table under the base interval. --- README.md | 4 +- crates/gitlawb-core/src/scan_token.rs | 100 +++-- crates/gitlawb-node/src/api/ipfs.rs | 465 +++++++++++++++++++++--- crates/gitlawb-node/src/auth/mod.rs | 3 +- crates/gitlawb-node/src/config.rs | 12 +- crates/gitlawb-node/src/ipfs_pin.rs | 183 ++++++++-- crates/gitlawb-node/src/main.rs | 7 +- crates/gitlawb-node/src/state.rs | 18 +- crates/gitlawb-node/src/test_support.rs | 250 ++++++++++--- 9 files changed, 878 insertions(+), 164 deletions(-) diff --git a/README.md b/README.md index 28827110..2377f756 100644 --- a/README.md +++ b/README.md @@ -353,8 +353,8 @@ Important node settings: | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | | `GITLAWB_IPFS_MAX_LEGACY_PROBES` | Max legacy (NULL-provenance) repos probed per `/ipfs/{cid}` request, bounding the scan-fallback fan-out. A truncated scan returns a retryable 503, not a false 404. Default 256. | -| `GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS` | Max repo rows one `/ipfs/{cid}` request's legacy scan may read from the database. The probe ceiling above only starts counting once a probe runs, and quarantined or private repos are denied before that, so this is what bounds a scan over an inventory that denies the caller everywhere. A truncated scan sheds a retryable 503 carrying an opaque `continuation` token; echoing it as `?scan=` resumes the scan, so a holder buried past the ceiling is served within `ceil(repos / ceiling) + 1` requests and no ceiling ever produces a 404. Every page is charged to the caller's `/ipfs` work allowance, so raising this raises that allowance too. Lowering it sharpens an oracle: because a truncation emits a token and a completed wrap does not, laddering to the end reveals the node's total repo count (private and quarantined included) to within one ceiling. Default 2048. | -| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. The effective cap is the tighter of this knob and the node's internal per-request history-walk ceiling of 17 (`MAX_PIN_SOURCES + 1`), so a value above 17 has no effect while a value below it does tighten the cap. Default 64. | +| `GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS` | Max repo rows one `/ipfs/{cid}` request's legacy scan may read from the database. The probe ceiling above only starts counting once a probe runs, and quarantined or private repos are denied before that, so this is what bounds a scan over an inventory that denies the caller everywhere. A truncated scan sheds a retryable 503 carrying an opaque `continuation` token; echoing it as `?scan=` resumes the scan where it stopped. Every per-request ceiling on this path (rows, probes, visits, retained rule bytes) mints one, so each request advances the ladder by at least the rows it read and a holder buried past a ceiling is reached in a bounded number of requests, `ceil(repos / ceiling) + 1` when this row ceiling is the one that binds. No ceiling ever produces a 404. Every page is charged to the caller's `/ipfs` work allowance, so raising this raises that allowance too. Lowering it sharpens an oracle: because a truncation emits a token and a completed wrap does not, laddering to the end reveals the node's total repo count (private and quarantined included) to within one ceiling. Default 2048. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks a `/ipfs/{cid}` request may run per phase; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. The effective cap is the tighter of this knob and the node's internal history-walk ceiling of 17 (`MAX_PIN_SOURCES + 1`), so a value above 17 has no effect while a value below it does tighten the cap. It is charged per phase: the provenance lookup and the legacy-scan fallback get separate equal budgets, so one request can run up to twice the cap in total. Default 64. | | `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate. Also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | | `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting and each also run under their own deadline (the lesser of `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` and the remaining budget), reaped via process-group teardown, so a hung `cat-file` cannot hold the request's walk slot past it. One hang path is still unbounded: the probe's object-store readability check is a plain filesystem sweep with nothing to reap, so a wedged filesystem can hold the slot past the deadline. Default 600. Accepted range is 1 to 3153600000 (100 years), since the node derives a deadline from this value and a larger one cannot be represented. | | `GITLAWB_IPFS_RESOLVE_BUDGET_SECS` | Shorter budget for the pre-walk CID resolve inside an admitted `/ipfs/{cid}` request: the lookup that maps the requested CID to its git oid(s), which runs while the scarce walk admission is already held. A well-formed CID with no pin row does no probe and no walk work, so without this it could hold a walk slot for the whole request budget while nothing walked, and enough such requests shed every real retrieval at admission. The effective deadline is the lesser of this and the remaining request budget, so a value above `GITLAWB_IPFS_REQUEST_BUDGET_SECS` degrades to the request budget. Only the resolve is on this clock; walk and probe work stay on the request budget, so a slow but progressing scan is never shed by it. Default 10. Accepted range is 1 to 3153600000 (100 years). | diff --git a/crates/gitlawb-core/src/scan_token.rs b/crates/gitlawb-core/src/scan_token.rs index 3ec55cd9..bc903b1d 100644 --- a/crates/gitlawb-core/src/scan_token.rs +++ b/crates/gitlawb-core/src/scan_token.rs @@ -17,17 +17,18 @@ //! * A fresh `OsRng` nonce on EVERY seal. Under a stream cipher a repeated nonce //! means repeated keystream, and an attacker who can force the node to seal a //! position whose plaintext they know XORs two tokens and recovers a withheld -//! row's fields in full — strictly worse than emitting plaintext. +//! row's fields in full, strictly worse than emitting plaintext. //! * FIXED-WIDTH plaintext. AEAD ciphertext is plaintext-length plus the tag, and //! both halves of a scan position vary in length, so a variable encoding would //! make token LENGTH a side channel for the sealed row (a short name under a //! short owner vs a long one). Every token this module mints is byte-identical -//! in length. +//! in length. The two halves are padded to their own separate widths, which is +//! a per-field constant and so still leaks nothing about a given row. //! * The canonical CID as associated data, so a token minted while scanning for //! one CID does not authenticate when replayed against another. //! -//! Every failure to open — wrong key, tampered bytes, wrong CID, expired, malformed -//! — returns the same `None`. The caller treats that as "no token" and starts at the +//! Every failure to open (wrong key, tampered bytes, wrong CID, expired, malformed) +//! returns the same `None`. The caller treats that as "no token" and starts at the //! front, so no failure class is distinguishable and the token is no oracle. use base64::{engine::general_purpose::URL_SAFE_NO_PAD as B64URL, Engine}; @@ -48,18 +49,38 @@ pub struct ScanPosition { /// Plaintext version byte, so a future layout change is a clean open-failure /// (treated as absent) rather than a misparse. -const VERSION: u8 = 1; +/// +/// Bumped to 2 when the two halves stopped sharing one width (see [`ID_WIDTH`]). +/// A token minted under the version-1 layout is a different length and a different +/// framing, so it opens to `None` and the caller restarts at the front, which is the +/// safe direction: a misparse would resume at a fabricated row and skip coverage. +const VERSION: u8 = 2; -/// Byte width each variable-length field is padded to. Both halves of a scan -/// position are stored at this width regardless of content, which is what keeps -/// every minted token the same length. A repo id is `/`, so 128 -/// clears a `did:key` z-base58 owner plus a long name with room to spare; anything -/// past it fails the seal loudly rather than silently truncating a cursor (a -/// truncated cursor would resume at the wrong row and skip coverage). -const FIELD_WIDTH: usize = 128; +/// Byte width the `created_at` half is padded to. Every value stored here is a +/// serialized timestamp, about 30 bytes, so 64 is roomy for the field's whole domain. +/// It is deliberately NOT widened to match [`ID_WIDTH`]: padding both halves to the id +/// width would nearly double every token for a field that can never use the space. +const CREATED_WIDTH: usize = 64; -/// `version | created_len:u16 | created[FIELD_WIDTH] | id_len:u16 | id[FIELD_WIDTH] | expires:i64` -const PLAINTEXT_LEN: usize = 1 + 2 + FIELD_WIDTH + 2 + FIELD_WIDTH + 8; +/// Byte width the `id` half is padded to. +/// +/// The bound is set by the WRITERS, not by what a typical id happens to look like. +/// `upsert_mirror_repo` builds `repos.id` as `{owner}/{name}`, and the slug validators +/// in the node's `repo_store` admit an owner of up to 255 bytes and a name of up to +/// 100, so 356 bytes is reachable through the ordinary write path and repo names are +/// peer-controllable. 384 clears that with margin. +/// +/// Under-sizing this is not a cosmetic bug. A row at a truncation boundary whose id +/// exceeds the width fails the seal, the handler sheds a 503 with no continuation, and +/// a tokenless shed is byte-identical to the wrapped-scan response whose contract is +/// "the absence of a token means the ladder is over". The boundary row is deterministic +/// for a stable inventory, so every retry reproduces it and every row past it becomes +/// permanently unreachable. Anything past the width still fails loudly rather than +/// silently truncating a cursor into one that resumes at the wrong row. +const ID_WIDTH: usize = 384; + +/// `version | created_len:u16 | created[CREATED_WIDTH] | id_len:u16 | id[ID_WIDTH] | expires:i64` +const PLAINTEXT_LEN: usize = 1 + 2 + CREATED_WIDTH + 2 + ID_WIDTH + 8; /// Nonce width for XChaCha20-Poly1305. const NONCE_LEN: usize = 24; @@ -73,33 +94,38 @@ pub fn new_key() -> [u8; 32] { // The two halves below are deliberately separate and adjacent. The framing pair owns // "every token is the same length"; the AEAD pair owns "the contents are confidential -// and CID-bound". Keeping them apart is what lets each property be exercised — and -// broken — without disturbing the other. +// and CID-bound". Keeping them apart is what lets each property be exercised (and +// broken) without disturbing the other. /// Encode a position into the FIXED-WIDTH plaintext: -/// `version | created_len:u16 | created[FIELD_WIDTH] | id_len:u16 | id[FIELD_WIDTH] | expires:i64` +/// `version | created_len:u16 | created[CREATED_WIDTH] | id_len:u16 | id[ID_WIDTH] | expires:i64` /// /// The padding is the point. AEAD ciphertext is plaintext-length plus the tag, and both /// halves of a scan position vary in length, so a length-prefixed encoding with no -/// padding would make token LENGTH a side channel for the sealed row. +/// padding would make token LENGTH a side channel for the sealed row. Each half is +/// padded to its OWN fixed width, which keeps every minted token the same length while +/// letting the id half carry the range the write path actually admits. fn encode_position(pos: &ScanPosition, expires_at_unix: i64) -> anyhow::Result> { let mut out = vec![0u8; PLAINTEXT_LEN]; out[0] = VERSION; let mut at = 1; - for field in [pos.created_at_key.as_bytes(), pos.id.as_bytes()] { - if field.len() > FIELD_WIDTH { + for (field, width) in [ + (pos.created_at_key.as_bytes(), CREATED_WIDTH), + (pos.id.as_bytes(), ID_WIDTH), + ] { + if field.len() > width { // Loud rather than truncating: a clipped cursor resumes at the wrong row and // silently skips coverage, which is the availability half of the bug this // token exists to fix. anyhow::bail!( - "scan token field is {} bytes, over the {FIELD_WIDTH}-byte fixed width", + "scan token field is {} bytes, over the {width}-byte fixed width", field.len() ); } out[at..at + 2].copy_from_slice(&(field.len() as u16).to_le_bytes()); at += 2; out[at..at + field.len()].copy_from_slice(field); - at += FIELD_WIDTH; + at += width; } out[at..at + 8].copy_from_slice(&expires_at_unix.to_le_bytes()); Ok(out) @@ -112,14 +138,14 @@ fn decode_position(bytes: &[u8]) -> Option<(ScanPosition, i64)> { } let mut at = 1; let mut fields = [const { String::new() }; 2]; - for slot in fields.iter_mut() { + for (slot, width) in fields.iter_mut().zip([CREATED_WIDTH, ID_WIDTH]) { let len = u16::from_le_bytes([bytes[at], bytes[at + 1]]) as usize; at += 2; - if len > FIELD_WIDTH { + if len > width { return None; } *slot = String::from_utf8(bytes[at..at + len].to_vec()).ok()?; - at += FIELD_WIDTH; + at += width; } let expires_at = i64::from_le_bytes(bytes[at..at + 8].try_into().ok()?); let [created_at_key, id] = fields; @@ -132,7 +158,7 @@ fn seal_bytes(key: &[u8; 32], cid: &str, plaintext: &[u8]) -> anyhow::Result Option> { /// Seal `pos` under `key`, bound to `cid`, expiring at `expires_at_unix`. /// /// Returns the base64url (no pad) token. Errors only when a field exceeds -/// [`FIELD_WIDTH`] or the AEAD itself fails — never silently truncates. +/// its half's fixed width ([`CREATED_WIDTH`], [`ID_WIDTH`]) or the AEAD itself fails, +/// never silently truncates. pub fn seal_scan_token( key: &[u8; 32], cid: &str, @@ -256,10 +283,27 @@ mod tests { ); } + /// The id half must clear the LARGEST repo id the node's own write path admits, + /// not merely a typical one. `upsert_mirror_repo` builds `repos.id` as + /// `{owner}/{name}`, and the slug validators in `repo_store` admit 255 bytes of + /// owner and 100 of name, so 356 is reachable. A width under that turns the + /// boundary row into a seal failure, which sheds a tokenless 503 that is + /// byte-identical to "your ladder is over" and strands every row past it forever. + #[test] + fn round_trips_a_repo_id_at_the_write_paths_maximum() { + let key = new_key(); + let id = format!("{}/{}", "o".repeat(255), "n".repeat(100)); + assert_eq!(id.len(), 356, "255 owner + '/' + 100 name"); + let p = pos("2020-01-01T00:00:03+00:00", &id); + let t = seal_scan_token(&key, "bafkcid", &p, 1 << 40) + .expect("a repo id the write path admits must seal, never fail the width"); + assert_eq!(open_scan_token(&key, "bafkcid", &t, 0), Some(p)); + } + #[test] fn a_field_over_the_fixed_width_fails_loudly() { let key = new_key(); - let p = pos("2020-01-01T00:00:03+00:00", &"x".repeat(FIELD_WIDTH + 1)); + let p = pos("2020-01-01T00:00:03+00:00", &"x".repeat(ID_WIDTH + 1)); assert!( seal_scan_token(&key, "bafkcid", &p, 1 << 40).is_err(), "an over-wide field must fail the seal, never be truncated into a cursor \ diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 95f50b0b..8fc48ad8 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -97,8 +97,8 @@ pub(crate) const LEGACY_SCAN_PAGE_ROWS: usize = 128; /// Hard per-request ceiling on how many repo ROWS the legacy scan's pager may fetch /// (#173 round 13, F2, INV-10). The probe ceiling above only starts counting once a -/// probe runs, and the two denial classes that dominate a hostile inventory — -/// quarantine and a root-scope visibility deny — return before either `walk.probes` +/// probe runs, and the two denial classes that dominate a hostile inventory +/// (quarantine and a root-scope visibility deny) return before either `walk.probes` /// or `walk.visits` increments. So an all-quarantined or all-root-denying node paged /// through its ENTIRE repo table at zero probes, anonymously, retaining every row and /// rule set, while holding one of the scarce global walk permits for up to the whole @@ -117,22 +117,29 @@ pub(crate) const LEGACY_SCAN_PAGE_ROWS: usize = 128; /// Tuning DOWN has a cost worth stating: token presence is a coarse inventory-size /// oracle. A ceiling truncation emits a token; a wrapped scan does not, so laddering /// until the `scan-wrapped` taint tells an anonymous caller the node's TOTAL repo -/// count — private and quarantined rows included — to within one ceiling. At the 2048 +/// count (private and quarantined rows included) to within one ceiling. At the 2048 /// default that is tolled and coarse; it sharpens as the ceiling is lowered. /// /// Tunable via `GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS` / `AppState`. pub(crate) const MAX_LEGACY_SCAN_ROWS_PER_REQUEST: usize = 2048; -/// Hard per-request ceiling on how many visibility RULES the legacy scan's pager may -/// retain (#173 round 13, F2, INV-10). The row ceiling bounds the row count but not -/// the memory each row drags in: `fetch_next_page` keeps every fetched page's rules in +/// Hard per-request ceiling on the BYTES of visibility rules the legacy scan's pager may +/// retain (#173 round 13, F2, INV-10). The row ceiling bounds the row count but not the +/// memory each row drags in: `fetch_next_page` keeps every fetched page's rules in /// `LegacyScanPager::rules` for the whole request (a later oid candidate re-reads them -/// rather than re-querying), so a node whose repos each carry hundreds of path-scoped -/// rules is retained-memory-unbounded at a row count well under the row ceiling. -/// Counting the retained rules and stopping on them is the second half of the same -/// bound. Not an operator knob: it is a memory guard, not a reach/coverage tradeoff, -/// and 8192 is four rules per row at the default row ceiling. -pub(crate) const MAX_LEGACY_SCAN_RULES_PER_REQUEST: usize = 8192; +/// rather than re-querying), so a node whose repos each carry many path-scoped rules is +/// retained-memory-unbounded at a row count well under the row ceiling. +/// +/// Bytes, not a rule count, because a count is the wrong unit for a memory bound: an +/// owner controls how many rules their repos carry AND how long each rule's +/// `reader_dids` list is, so a handful of rules can retain as much as thousands. The +/// check runs inside `fetch_next_page` immediately after the rules query returns, so the +/// page that blows the budget truncates the request that bought it. +/// +/// Not an operator knob: it is a memory guard, not a reach/coverage tradeoff. 4 MiB is +/// about 2 KiB per row at the default row ceiling, which is a generous rule set per repo +/// and still a bounded allocation for one anonymous GET. +pub(crate) const MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST: usize = 4 * 1024 * 1024; /// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` buffers and serves /// (#173 round 8, F6, INV-10). The serve reads via a blocking `git cat-file` and @@ -181,8 +188,30 @@ struct LegacyScanPager { /// `rows.len()`, which is the same number today but would silently stop tracking /// the DB-facing cost if the pager ever dropped gated rows. fetched_rows: usize, - /// Visibility rules retained this request, the quantity the rules ceiling bounds. - fetched_rules: usize, + /// Bytes of visibility rules retained this request, the quantity the rules ceiling + /// bounds. + fetched_rule_bytes: usize, + /// Set by `fetch_next_page` when the page it just retained put + /// `fetched_rule_bytes` over the ceiling. The flag exists because the check has to + /// happen where the retention happens: measuring only when another page is + /// contemplated lets the page that actually blew the budget go unnoticed on a scan + /// that ends there. + rule_bytes_exceeded: bool, +} + +/// Retained size of one visibility rule, in bytes. +/// +/// The heap the pager holds for a rule is its owned strings, and the one an owner can +/// grow without limit is `reader_dids` (there is no per-repo rule cap and no per-rule +/// reader cap). Counting the strings rather than the struct is what makes this track +/// the thing that can actually get large; the fixed fields are noise beside a long +/// reader list. +fn rule_retained_bytes(rule: &crate::db::VisibilityRule) -> usize { + rule.id.len() + + rule.repo_id.len() + + rule.path_glob.len() + + rule.created_by.len() + + rule.reader_dids.iter().map(String::len).sum::() } impl LegacyScanPager { @@ -260,7 +289,24 @@ impl LegacyScanPager { return Err(budget_shed()); } }; - self.fetched_rules += rules.values().map(Vec::len).sum::(); + // Measure the page HERE, where it is retained, not on the next trip round the + // caller's loop. A rules query answers with whatever the matched repos carry; + // nothing bounds that per repo, so one page can be arbitrarily large and a check + // that only runs when another page is contemplated never sees it on a scan that + // ends on this one. Bytes rather than a rule count for the same reason: the + // quantity that grows is the length of each `reader_dids` list, not the number + // of rows in `visibility_rules`. + self.fetched_rule_bytes += rules + .values() + .flat_map(|v| v.iter()) + .map(rule_retained_bytes) + .sum::(); + // Only a page with more behind it truncates. A short page has already retained + // everything there is, so stopping on it would turn a scan that genuinely + // covered the table into a permanent 503 for an object that is simply absent. + if !self.exhausted && self.fetched_rule_bytes >= state.ipfs_max_legacy_scan_rule_bytes { + self.rule_bytes_exceeded = true; + } self.rules.extend(rules); self.rows.extend(page); Ok(()) @@ -590,8 +636,8 @@ pub async fn get_by_cid( // Resume from the caller's sealed continuation, if they sent one that opens. The // node holds NO scan state of its own: the position rides in the token, which is // what keeps concurrent ladders from advancing or resetting each other. Every - // failure class — tampered, wrong key (a prior boot's), expired, malformed, minted - // for a different CID — lands on the same `None` and starts at the front, silently, + // failure class (tampered, a prior boot's key, expired, malformed, minted for a + // different CID) lands on the same `None` and starts at the front, silently, // so no probe distinguishes them (INV-13). if let Some(token) = scan_query.scan.as_deref() { if let Some(pos) = gitlawb_core::scan_token::open_scan_token( @@ -840,17 +886,29 @@ pub async fn get_by_cid( // Stopping at any of these leaves every unread repo unproven, so // each TAINTS: the tail sheds a retryable 503 naming the ceiling, // never a definitive 404 (#173, F2). + // + // All four breaks fire at `idx == pager.rows.len()`, so + // `pager.cursor` is the same well-defined resume boundary in every + // arm and every one of them mints a continuation. These two are not + // an afterthought to the two below: the probe and visit ceilings + // BIND FIRST on any inventory carrying root-readable repos, long + // before the far larger row ceiling, so a tokenless break here is + // the common case rather than the rare one, and a tokenless shed is + // byte-identical to the wrapped-scan answer that tells the caller + // their ladder is over. if walk.probes >= state.ipfs_max_legacy_probes { walk.taint("probe-ceiling"); + scan_continuation = pager.cursor.clone(); break; } if walk.visits >= state.config.ipfs_max_repo_visits { walk.taint("visit-ceiling"); + scan_continuation = pager.cursor.clone(); break; } // Row ceiling (F2). The two checks above only bind once a probe or a // visit has been spent, and the gate returns Skip on quarantine and - // on a root-scope deny BEFORE either counter moves — so an + // on a root-scope deny BEFORE either counter moves, so an // all-denying inventory paged the node's whole repo table at zero // probes, anonymously, while holding a scarce walk permit. This is // the check that actually stops that scan. @@ -859,17 +917,19 @@ pub async fn get_by_cid( scan_continuation = pager.cursor.clone(); break; } - // Rules ceiling: the row ceiling bounds rows, not the rules each row - // drags in, and the pager retains every fetched page's rules for the - // whole request. - if pager.fetched_rules >= state.ipfs_max_legacy_scan_rules { + // Rule-bytes ceiling: the row ceiling bounds rows, not the rules each + // row drags in, and the pager retains every fetched page's rules for + // the whole request. The decision itself was taken inside + // `fetch_next_page`, against the page that did the retaining, so the + // request that bought an oversized page is the one that truncates. + if pager.rule_bytes_exceeded { walk.taint("rules-ceiling"); scan_continuation = pager.cursor.clone(); break; } // Page toll (F2). Every page is work bought by an anonymous caller, - // so it is charged to the per-IP WORK bucket — the same bucket the - // per-probe charge debits — immediately before the query it pays + // so it is charged to the per-IP WORK bucket (the same bucket the + // per-probe charge debits) immediately before the query it pays // for. Without it a denial-only inventory could be re-paged for free // by re-requesting, which is the across-request half of the same // amplification. Reuses the `source_key` already resolved at @@ -929,9 +989,9 @@ pub async fn get_by_cid( // // The condition is evaluated HERE, on `pager.exhausted`, and deliberately not at any // particular break site. That is what covers the degenerate zero-row resume: a token - // at or past the last row — which the row ceiling emits whenever the row count is an + // at or past the last row (which the row ceiling emits whenever the row count is an // exact multiple of the ceiling, and which repo deletion between ladder steps also - // reaches — fetches an EMPTY short page, sets `exhausted`, and breaks without + // reaches) fetches an EMPTY short page, sets `exhausted`, and breaks without // gating anything. An implementation keying this on having fetched a page passes // every other case and turns exactly that incomplete search into a false 404. // @@ -982,7 +1042,7 @@ pub async fn get_by_cid( if !walk.truncated_by.is_empty() { // Seal the continuation HERE, the single mint site. The position is the last // row the pager FETCHED, and on a scan that served nothing every fetched row is - // by construction private or quarantined — so its `created_at` and its `id` + // by construction private or quarantined, so its `created_at` and its `id` // (which carries the owner's DID) are withheld fields and the token must be // confidential, not merely tamper-evident (INV-13). A seal failure is not fatal // to the shed: drop the continuation and answer the plain 503, which degrades to @@ -1004,7 +1064,7 @@ pub async fn get_by_cid( }); return Err(AppError::SearchIncomplete { message: format!( - "CID {cid_str} search incomplete ({}) — retry", + "CID {cid_str} search incomplete ({}); retry", walk.truncated_by.join("+") ), continuation, @@ -2150,7 +2210,7 @@ mod tests { /// Seed `n` PRIVATE repos owned by a foreign DID, in scan order, with `rules_each` /// path-scoped rules apiece. An anonymous caller is denied at the root gate on every - /// one, and a root deny costs neither a probe nor a visit — which is exactly the + /// one, and a root deny costs neither a probe nor a visit, which is exactly the /// hole the row ceiling closes. Their `disk_path`s do not exist on purpose: if a /// deny ever stopped short-circuiting, the missing-dir probe would taint the scan /// with a different source and the tests' taint assertions would catch it. @@ -3849,7 +3909,7 @@ mod tests { ); assert!( continuation_of(&wrapped).is_none(), - "a wrapped scan emits NO token — there is nothing left to resume: {wrapped}" + "a wrapped scan emits NO token, since there is nothing left to resume: {wrapped}" ); // Leg 3: the 404 tail stays reachable for a front-started scan that exhausts @@ -3877,7 +3937,7 @@ mod tests { /// /// Ceiling 4 over 10 denial rows with the public holder behind them: the bound is /// `ceil(10 / 4) + 1 = 4` requests. Every intermediate response is the retryable - /// 503 with a token, and no 429 interrupts the ladder — which is what the floor fix + /// 503 with a token, and no 429 interrupts the ladder, which is what the floor fix /// pins. The work bucket is sized to the DERIVED floor of a config whose page term /// dominates (probe knob 1, row knob 896 = 7 pages, so floor = 8); under the old /// floor (`max(route, probes)` = 1) the very first page would 429. @@ -3894,7 +3954,7 @@ mod tests { /// /// This test has NO pre-fix RED, and that is by design rather than an omission. /// Mutation A (delete the row ceiling) must leave it GREEN, which means its - /// assertions have to tolerate the holder being served on the very first request — + /// assertions have to tolerate the holder being served on the very first request, /// exactly what an unbounded scan does. So the pre-fix head passes it. Its /// load-bearing proof is mutation C, its designated mutant: C keeps the ceiling and /// keeps minting tokens but never honours one, which is the only shape that makes @@ -3988,15 +4048,257 @@ mod tests { ); } + /// Seed `n` PUBLIC (root-READABLE) mirror rows in scan order, with disk paths that + /// do not exist. + /// + /// The distinction from `seed_root_denying_repos` is the whole point: a private row + /// is denied at the root gate before `walk.probes` moves, so a denial-only fixture + /// can never reach the probe or visit ceilings. These rows pass the root gate, so + /// each one is CHARGED a probe, which is what drives the pager to the probe ceiling. + async fn seed_root_readable_repos(state: &crate::state::AppState, prefix: &str, n: usize) { + for i in 0..n { + state + .db + .upsert_mirror_repo( + &format!("z6readable{prefix}"), + &format!("{prefix}-{i:04}"), + &format!("/nonexistent/{prefix}-{i:04}"), + None, + false, + ) + .await + .expect("seed a root-readable mirror row"); + } + } + + /// The continuation must survive a repo id at the WRITE PATH's maximum. + /// + /// `repos.id` is `{owner}/{name}`, and the node's own slug validators admit 255 + /// bytes of owner and 100 of name, so a 356-byte id is reachable and repo names are + /// peer-controllable. When such a row lands on a truncation boundary the seal is the + /// only thing standing between it and a tokenless 503, and a tokenless 503 is + /// byte-identical to the wrapped-scan answer whose contract is "your ladder is + /// over". The boundary row is deterministic for a stable inventory, so every retry + /// reproduces it and every row past it is permanently unreachable. + /// + /// MUTATION (RED): narrow the token's id width back to 128 and the shed loses its + /// continuation. + #[sqlx::test] + async fn get_by_cid_row_ceiling_continuation_survives_a_max_length_repo_id(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 1; + state.ipfs_max_legacy_scan_rows = 1; + + let owner = format!("did:key:{}", "z".repeat(247)); + assert_eq!( + owner.len(), + 255, + "the largest owner the slug validator admits" + ); + let name = "n".repeat(100); + let at = scan_order_stamp(0); + state + .db + .create_repo(&crate::db::RepoRecord { + id: format!("{owner}/{name}"), + name: name.clone(), + owner_did: owner.clone(), + description: None, + is_public: false, + default_branch: "main".to_string(), + created_at: at, + updated_at: at, + disk_path: "/nonexistent/max-length-id".to_string(), + forked_from: None, + machine_id: None, + }) + .await + .expect("seed the boundary row"); + + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let peer: SocketAddr = "203.0.113.150:5000".parse().unwrap(); + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + assert_eq!(body["error"], "search_incomplete", "{body}"); + assert!( + continuation_of(&body).is_some(), + "a truncation on a row whose id the write path admits must still carry a \ + continuation; without one the ladder ends here forever: {body}" + ); + } + + /// The PROBE ceiling must advance the ladder, not end it. + /// + /// `ipfs_max_legacy_probes` binds first on any inventory containing root-readable + /// repos, long before the row ceiling that does mint a token. A probe-ceiling break + /// with no continuation makes the shed tokenless, which reads to the caller as "the + /// ladder is over", so a holder past the probe ceiling is unreachable on every + /// retry. + /// + /// The fixture seeds ROOT-READABLE rows on purpose: every other scan test in this + /// file uses `seed_root_denying_repos`, and a root deny returns before a probe is + /// charged, which is exactly why the shipped suite could not see this. + /// + /// MUTATION (RED): drop the continuation from the probe-ceiling arm and the ladder + /// never reaches the holder. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_probe_ceiling_ladders_to_a_holder_past_it(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + // The row ceiling is deliberately far out of reach: the probe ceiling is what + // must stop this scan, and it is what must carry the ladder forward. + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + // Generous so no 429 interrupts an honest caller's ladder; the toll is covered + // by its own test. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_readable_repos(&state, "probe", 6).await; + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6probe", + "holder", + b"past the probes\n", + ) + .await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.151:5000".parse().unwrap(); + let bound = 6usize.div_ceil(2) + 1; + + let mut token: Option = None; + let mut served_at = None; + for step in 1..=bound { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + assert_eq!(body["error"], "search_incomplete", "{body}"); + token = Some(continuation_of(&body).unwrap_or_else(|| { + panic!( + "the probe-ceiling shed at step {step} must carry a continuation; \ + a tokenless shed is indistinguishable from a finished ladder: {body}" + ) + })); + } + assert!( + served_at.is_some(), + "a holder past the probe ceiling must be reached within {bound} \ + token-echoing requests, not stranded forever" + ); + } + + /// The VISIT ceiling must advance the ladder too, for the same reason as the probe + /// ceiling: it is the sibling arm, it fires on the same root-readable inventory, and + /// a tokenless shed there strands everything behind it just as permanently. + /// + /// MUTATION (RED): drop the continuation from the visit-ceiling arm. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_visit_ceiling_ladders_to_a_holder_past_it(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + // Probes must NOT bind: the visit ceiling is the one under test. + state.ipfs_max_legacy_probes = 1024; + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repo_visits = 2; + state.config = std::sync::Arc::new(cfg); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_readable_repos(&state, "visit", 6).await; + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6visit", + "holder", + b"past the visits\n", + ) + .await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.153:5000".parse().unwrap(); + let bound = 6usize.div_ceil(2) + 1; + + let mut token: Option = None; + let mut served_at = None; + for step in 1..=bound { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + token = Some(continuation_of(&body).unwrap_or_else(|| { + panic!( + "the visit-ceiling shed at step {step} must carry a continuation: \ + {body}" + ) + })); + } + assert!( + served_at.is_some(), + "a holder past the visit ceiling must be reached within {bound} \ + token-echoing requests, not stranded forever" + ); + } + /// Scenario 6: the page toll accumulates ACROSS requests. /// /// Every page the scan buys is charged to the caller's per-IP work bucket, so a /// denial-only inventory cannot be re-paged for free by re-requesting. A bucket /// sized to 4 pages admits four requests' worth of paging and then sheds the fifth - /// with 429 — buying NO page (the `preload_queries()` count stalls) and carrying NO + /// with 429, buying NO page (the `preload_queries()` count stalls) and carrying NO /// token. The caller's PREVIOUS token still resumes them once the bucket refills. /// - /// MUTATION D (RED): drop the page toll and the pages are free again — the fifth + /// MUTATION D (RED): drop the page toll and the pages are free again, so the fifth /// request buys its page and never 429s. #[sqlx::test] async fn get_by_cid_denial_only_requests_throttle_across_requests(pool: sqlx::PgPool) { @@ -4052,7 +4354,7 @@ mod tests { assert_eq!( crate::api::ipfs::preload_queries(), pages_before, - "and the braked request must buy NO page — a 429 that still paged would \ + "and the braked request must buy NO page; a 429 that still paged would \ leave the amplification exactly where it was" ); assert!( @@ -4062,7 +4364,7 @@ mod tests { ); // Bucket refilled (a fresh limiter is the window elapsing). The token the caller - // already holds still resumes them — the throttle cost them a page, not their + // already holds still resumes them: the throttle cost them a page, not their // place in the ladder. let mut refilled = state.clone(); refilled.ipfs_work_rate_limiter = @@ -4083,7 +4385,7 @@ mod tests { assert_eq!( crate::api::ipfs::scan_rows(), 2, - "and it must resume at the sealed position — one ceiling's worth of rows \ + "and it must resume at the sealed position, one ceiling's worth of rows \ read, not a restart at the front" ); } @@ -4100,7 +4402,7 @@ mod tests { state.ipfs_legacy_scan_page_rows = 2; // Rows are NOT the binding ceiling here. state.ipfs_max_legacy_scan_rows = 1000; - state.ipfs_max_legacy_scan_rules = 3; + state.ipfs_max_legacy_scan_rule_bytes = 3; seed_root_denying_repos(&state, "rules", 8, 2).await; let cid = seed_legacy_pin(&state, &absent_oid()).await; @@ -4139,12 +4441,77 @@ mod tests { ); } + /// A SINGLE page whose rules exceed the ceiling truncates the request that bought + /// it. + /// + /// The ceiling bounds retained MEMORY, and the thing it has to bound is bytes: there + /// is no per-repo cap on `visibility_rules`, and an owner controls both how many + /// rules their repos carry and how long each `reader_dids` list is. Counted in rules + /// and checked only between pages, one page could carry arbitrarily many bytes and + /// the guard would not notice until it was asked for the NEXT page, which on a scan + /// that ends there is never. + /// + /// The fixture is calibrated so page one alone clears the byte ceiling while its + /// four rules are far under any plausible rule COUNT, which is what makes the unit + /// the thing under test. + /// + /// MUTATION (RED): move the check back between pages and the request that bought the + /// oversized page runs on to a clean 404. + #[sqlx::test] + async fn get_by_cid_one_oversized_page_truncates_its_own_request(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + // Neither rows nor probes may bind: this is the rule-bytes guard alone. + state.ipfs_max_legacy_scan_rows = 1000; + // Under the byte ceiling one page (2 rows x 2 rules, each rule carrying its repo + // id, its glob and a reader DID) is already over. Under a RULE count of 200 that + // same page is four. + state.ipfs_max_legacy_scan_rule_bytes = 200; + seed_root_denying_repos(&state, "bytes", 6, 2).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let peer: SocketAddr = "203.0.113.152:5000".parse().unwrap(); + crate::api::ipfs::reset_scan_rows(); + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the page that blew the retained-byte ceiling must truncate its OWN request, \ + not run on to a 404: {body}" + ); + assert!( + body["message"] + .as_str() + .unwrap_or_default() + .contains("rules-ceiling"), + "the shed names the rule-bytes ceiling: {body}" + ); + assert_eq!( + crate::api::ipfs::scan_rows(), + 2, + "and it stops on the page that bought the bytes, not a page later" + ); + assert!( + continuation_of(&body).is_some(), + "a rule-bytes truncation carries a continuation like every other ceiling: \ + {body}" + ); + } + /// Scenario 8: interleaved callers stay isolated. Two source keys alternate /// token-echoing ladders against the same denial-heavy inventory with the holder /// past the ceiling; each must reach its own 200 within its own bound. /// - /// Isolation is STRUCTURAL under this design — each ladder's entire state rides in - /// its own tokens and the node holds none — so this is the executed confirmation + /// Isolation is STRUCTURAL under this design (each ladder's entire state rides in + /// its own tokens and the node holds none), so this is the executed confirmation /// rather than a mutant target. It is what rules out the rejected designs: a /// node-global persisted cursor lets these two advance each other's window, and a /// per-caller server-side map lets one evict the other. @@ -4279,8 +4646,8 @@ mod tests { /// A denial-only scan fetches nothing BUT withheld rows, so the row its token seals /// is by construction a private or quarantined repo the caller may not read. Its /// `created_at` leaks a hidden repo's creation time and its `id` carries the owner's - /// DID. Base64 is transport, not confidentiality — this is the exact shape #134 - /// shipped and INV-13 records — so the token must be AEAD-SEALED. + /// DID. Base64 is transport, not confidentiality (this is the exact shape #134 + /// shipped and INV-13 records), so the token must be AEAD-SEALED. /// /// The fixture is arranged so the row at the truncation boundary (the row the token /// seals) IS one of the poisoned withheld repos. Stated because it is load-bearing: @@ -4354,7 +4721,7 @@ mod tests { assert!( !decoded_text.contains(marker.as_str()), "the token's DECODED bytes must not carry a withheld repo's {what} \ - ({marker}) — base64 is transport, not confidentiality (INV-13)" + ({marker}); base64 is transport, not confidentiality (INV-13)" ); assert!( decoded @@ -4365,7 +4732,7 @@ mod tests { } } - // And the row it actually seals IS one of the poisoned withheld rows — checked + // And the row it actually seals IS one of the poisoned withheld rows, checked // after the leak assertions so a broken seal is reported as a leak, not as a // fixture failure. Load-bearing: seeding a READABLE repo at the boundary would // leave mutation E green and this whole guard would stop proving anything. @@ -4423,7 +4790,7 @@ mod tests { short.len(), long.len(), "tokens sealing rows of very different id lengths must be byte-identical in \ - length, or the length is a side channel for the withheld row — which the \ + length, or the length is a side channel for the withheld row, which the \ substring assertions above structurally cannot see" ); } @@ -4431,7 +4798,7 @@ mod tests { /// Scenario 10: tampered, foreign-CID, and expired tokens are ABSENT, uniformly. /// /// Each of the three failure classes must produce exactly the front-started response - /// a tokenless request gets: same status, same body shape, and — the decisive part — + /// a tokenless request gets: same status, same body shape, and (the decisive part) /// an emitted continuation sealing the FRONT window's last row, not the row the /// rejected token named. Never an error, never a resumed position, and no way to /// tell the three classes apart. @@ -4555,7 +4922,7 @@ mod tests { /// This is the property the whole confidentiality claim rests on and the one the /// other two token guards cannot see: both of them pass unchanged under a constant /// nonce. Under a stream cipher a repeated nonce repeats the keystream, so two - /// tokens sealed under one nonce XOR to the difference of their plaintexts — and an + /// tokens sealed under one nonce XOR to the difference of their plaintexts, and an /// attacker who can force the node to seal a position they know then recovers a /// withheld row's fields in full. That is strictly worse than the base64 defect /// INV-13 records. @@ -4580,7 +4947,7 @@ mod tests { assert_ne!( first, second, - "sealing the same position twice must produce different bytes — identical \ + "sealing the same position twice must produce different bytes; identical \ tokens mean a reused nonce, and a reused nonce under a stream cipher leaks \ the withheld plaintext to anyone holding two tokens" ); @@ -4631,7 +4998,7 @@ mod tests { assert_eq!( status, StatusCode::SERVICE_UNAVAILABLE, - "no rung of this ladder may 404 — least of all the zero-row one \ + "no rung of this ladder may 404, least of all the zero-row one \ (step {step}): {body}" ); match continuation_of(&body) { diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 30a20a10..b9e3f4b7 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -522,7 +522,8 @@ mod tests { ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, ipfs_max_legacy_scan_rows: crate::api::ipfs::MAX_LEGACY_SCAN_ROWS_PER_REQUEST, - ipfs_max_legacy_scan_rules: crate::api::ipfs::MAX_LEGACY_SCAN_RULES_PER_REQUEST, + ipfs_max_legacy_scan_rule_bytes: + crate::api::ipfs::MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST, ipfs_scan_token_key: Arc::new(crate::state::AppState::new_scan_token_key()), ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 54841a4c..00ab6edd 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -523,8 +523,8 @@ pub struct Config { /// absent with a 404. The handler still short-circuits the moment it serves. /// Must be between 1 and 1_048_576. Default: 64. /// - /// The effective per-request ceiling is the TIGHTER of this knob and the node's - /// internal per-request history-walk ceiling, `MAX_PIN_SOURCES + 1` = 17 (see + /// The effective ceiling is the TIGHTER of this knob and the node's internal + /// history-walk ceiling, `MAX_PIN_SOURCES + 1` = 17 (see /// `api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST` and the `min()` that combines the /// two in the resolver). Setting this above 17 changes nothing, because the /// internal ceiling already binds. Setting it below 17 does lower the cap: the @@ -532,6 +532,12 @@ pub struct Config { /// before its whole bounded provenance source set has been tried, so an operator /// who goes under it is choosing a tighter cap that can 503 a provenanced /// request, which is allowed. + /// + /// That combined cap is charged PER PHASE, not per request: the provenance phase and + /// the legacy-scan fallback each get their own equal budget, so one request can run + /// up to twice it in total (see `MAX_HISTORY_WALKS_PER_REQUEST`, which explains why + /// the split is what keeps the fallback from inheriting a provenance phase's spent + /// remainder). #[arg( long, env = "GITLAWB_IPFS_MAX_REPOS_WALKED", @@ -1035,7 +1041,7 @@ mod tests { "the work budget must clear one full legacy search per window" ); - // Tight route limit (1): the floor lifts the work budget to a full deep scan — + // Tight route limit (1): the floor lifts the work budget to a full deep scan, // the 256-probe budget PLUS the page toll a 2048-row ceiling costs at 128 rows // per page (16) = 272, NOT down to 1. A single deep search still completes its // full scan without self-throttling on either charge. diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 628d828c..71843942 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -309,12 +309,14 @@ pub(crate) struct SweepStats { /// Why a sweep run ended, which is what [`run_sweep_rearmed`] dispatches on. /// -/// Two of the three arms are re-armable and one is not. A run that walked to the end of -/// the table and a run that paused on [`MAX_DEAD_ROW_READS_PER_RUN`] both left the node -/// in a state a later run improves, so the wrapper sleeps and goes again. A failing pass -/// QUERY is a broken database, and retrying it on a timer would turn one logged failure -/// into an endless stream of them, so the wrapper returns and leaves the run one-shot, -/// exactly as it was before the re-arm existed. +/// All three arms are re-armable; what differs is how long the wrapper waits. A run that +/// walked to the end of the table and a run that paused on +/// [`MAX_DEAD_ROW_READS_PER_RUN`] both left the node in a state a later run improves, so +/// the wrapper sleeps and goes again. A failing pass QUERY is a broken database, so it +/// waits far longer (see [`SWEEP_REARM_DELAY`]) rather than turning one logged +/// failure into a stream of them, but it does go again: exiting for good made a single +/// deadlock or connection reset disable legacy-CID repair for the whole process +/// lifetime. #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] pub(crate) enum SweepStop { /// The ordered walk reached the end of the table (a short batch). @@ -496,7 +498,7 @@ async fn load_discovery_ctx( // exists to forbid. Same query, different threat model. This is background // maintenance on a timer: no caller to amplify, no permit to pin, and the pass needs // the whole warm candidate set before it can call a row settled. Do not "align" this - // loop with the resolver's — the budgets it stops on have no counterpart here. + // loop with the resolver's: the budgets it stops on have no counterpart here. // // Per PASS, not per row: `load_discovery_ctx` already runs once per pass and its // result is reused for every source-less row, so the paging cost is paid once. @@ -1112,24 +1114,74 @@ pub(crate) async fn sweep_legacy_provider_cids( /// a few queries every five minutes and the migration still converges in hours rather /// than never. It is also the anti-hot-spin floor for the degenerate case, an empty or /// fully repaired table where a run returns immediately. +/// +/// That pricing holds for a table that settles. It does NOT hold for the table that +/// never does: a node carrying rows whose source bytes are permanently gone spends up to +/// [`MAX_DEAD_ROW_READS_PER_RUN`] (64) object reads on every run, repairs nothing, and +/// arrives back at exactly the same rows next time. At this interval alone that is 64 +/// fruitless `git cat-file` invocations every five minutes for the life of the process. +/// This constant is therefore the interval after a run that REPAIRED something; +/// [`SWEEP_IDLE_REARM_MULTIPLIER`] is what a run that repaired nothing backs off to (one +/// hour), and it is what keeps the unrepairable case from costing that forever. A failed +/// pass waits [`SWEEP_FAILURE_REARM_MULTIPLIER`] times this (30 minutes). pub(crate) const SWEEP_REARM_DELAY: Duration = Duration::from_secs(300); -/// Run the legacy provider-CID sweep on a timer until shutdown or a broken database. +/// How much longer the sweep waits after a run that repaired NOTHING, as a multiple of +/// the base interval. +/// +/// The base interval above is priced against a settled table, where a run is an indexed +/// range scan and a codec decode per row. It is not priced against the case that never +/// settles: a node carrying rows whose source bytes are permanently gone spends up to +/// [`MAX_DEAD_ROW_READS_PER_RUN`] object reads on every run, repairs nothing, and does +/// it again on the next one, forever. At the base interval that is 64 fruitless object +/// reads every five minutes, for the life of the process, against a table that will +/// never repair. +/// +/// So a run that repaired nothing backs off to the longer interval instead. Any run that +/// repairs at least one row resets to the base, because a table still yielding repairs +/// is one worth walking often. A single longer interval, not an exponential ladder: the +/// point is to stop paying a fixed waste every five minutes. +/// +/// Expressed as a multiple of the base rather than as an absolute so that shortening the +/// base (which the wrapper's tests do) shortens all three intervals coherently. +/// One hour is what it comes to in production. +const SWEEP_IDLE_REARM_MULTIPLIER: u32 = 12; + +/// How much longer the sweep waits after a pass QUERY failed, as a multiple of the base. +/// +/// A failing pass is a broken database, not a broken sweep, and retrying it on the base +/// interval would turn one fault into a stream of failing queries. But the alternative +/// the wrapper used to take, returning for good, is worse: one deadlock or connection +/// reset permanently disabled legacy-CID repair for the whole process lifetime, and +/// nothing joins the task, so the only trace was a single warn. Half an hour in +/// production is long enough not to hammer a database that is down, short enough that a +/// transient fault costs one window rather than a reboot. +const SWEEP_FAILURE_REARM_MULTIPLIER: u32 = 6; + +/// Consecutive failed runs before the per-failure log escalates from `warn!` to +/// `error!`. A single failure is a transient the next run recovers from; a standing +/// stream of them is a database that needs an operator, and at the production failure +/// interval this is reached in a couple of hours. +const SWEEP_FAILURE_ESCALATE_AFTER: u32 = 3; + +/// Run the legacy provider-CID sweep on a timer until shutdown. /// /// Owns the [`DiscoveryTraversalState`] across runs, which is the reason this is a /// wrapper and not a loop inside `sweep_legacy_provider_cids`: a run can PAUSE /// mid-traversal on [`MAX_DEAD_ROW_READS_PER_RUN`], and the traversal it was in is /// finished by a later run, which has to apply the advance the earlier run earned. /// -/// Sleeps `rearm_delay` after EVERY re-armable run, unconditionally. Not conditional on -/// the run having done work: a run over an empty or fully repaired table returns +/// Sleeps after EVERY run, unconditionally, at the interval its outcome earns: +/// `rearm_delay` after a run that repaired something, that scaled by +/// [`SWEEP_IDLE_REARM_MULTIPLIER`] after one that repaired nothing, and by +/// [`SWEEP_FAILURE_REARM_MULTIPLIER`] after a failed pass query. Not conditional on the +/// run having done work: a run over an empty or fully repaired table returns /// immediately, and without the sleep this loop would spin the database as fast as it /// can answer. /// -/// Returns only on [`SweepStop::PassFailed`], preserving the one-shot behavior a failing -/// database had before the re-arm existed. On a healthy node it never returns, which is -/// why the per-run summary is logged HERE rather than by the caller off the awaited -/// value. +/// It NEVER returns, which is why it yields nothing: shutdown preempts it from the +/// outside, through the `tokio::select!` the caller wraps it in, so there is no awaited +/// value for a caller to log and the per-run summary is logged HERE. pub(crate) async fn run_sweep_rearmed( repos_dir: &std::path::Path, git_bin: &str, @@ -1138,9 +1190,9 @@ pub(crate) async fn run_sweep_rearmed( delay: Duration, rearm_delay: Duration, db: &crate::db::Db, -) -> SweepStats { - let mut totals = SweepStats::default(); +) { let mut traversal = DiscoveryTraversalState::default(); + let mut consecutive_failures: u32 = 0; loop { let run = sweep_legacy_provider_cids( repos_dir, @@ -1161,20 +1213,80 @@ pub(crate) async fn run_sweep_rearmed( "legacy provider-CID sweep run finished" ); } - totals.scanned += run.scanned; - totals.repaired += run.repaired; - totals.passes += run.passes; - totals.retryable_skips += run.retryable_skips; - totals.dead_row_reads += run.dead_row_reads; - totals.discovery_budget_spent |= run.discovery_budget_spent; - totals.stop = run.stop; - if run.stop == SweepStop::PassFailed { - return totals; - } - tokio::time::sleep(rearm_delay).await; + #[cfg(test)] + note_sweep_run(); + + // A failed pass RE-ARMS, on its own longer interval, and never returns. Returning + // was the whole defect: the wrapper exists so coverage is wall-clock rather than + // a reboot count, and one deadlock or connection reset used to disable + // legacy-CID repair for the entire process lifetime. Nothing joins this task, so + // the only trace was a single warn and the node quietly kept withholding every + // unrepaired row. The longer interval is what keeps a genuinely broken database + // from being hammered, and the escalation is what keeps it from being quiet. + let next = if run.stop == SweepStop::PassFailed { + consecutive_failures = consecutive_failures.saturating_add(1); + if consecutive_failures > SWEEP_FAILURE_ESCALATE_AFTER { + tracing::error!( + consecutive_failures, + "legacy provider-CID sweep has failed every run for a while; the \ + database looks broken and legacy CID repair is not progressing" + ); + } else { + tracing::warn!( + consecutive_failures, + "legacy provider-CID sweep pass failed; re-arming on the longer \ + failure interval" + ); + } + rearm_delay.saturating_mul(SWEEP_FAILURE_REARM_MULTIPLIER) + } else { + consecutive_failures = 0; + if run.repaired == 0 { + // Nothing repaired: either the table is settled, or it holds rows that + // will never repair and this run just paid up to + // MAX_DEAD_ROW_READS_PER_RUN fruitless object reads to learn that + // again. Back off rather than pay it every base interval forever. Any + // run that does repair something resets to the base above. + rearm_delay.saturating_mul(SWEEP_IDLE_REARM_MULTIPLIER) + } else { + rearm_delay + } + }; + tokio::time::sleep(next).await; } } +// Test-only wrapper-loop seam: how many RUNS the re-arm loop has completed. The loop +// never returns, so a test cannot observe its behaviour off a return value, and the +// interval it chose is only visible as "did another run happen inside this window". +// A process-wide counter rather than a `thread_local`, because the loop is awaited on a +// multi-thread runtime and can move between threads; the sweep tests that read it +// serialize on `sweep_run_lock` so they never see each other's increments. +#[cfg(test)] +static SWEEP_RUNS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +#[cfg(test)] +fn note_sweep_run() { + SWEEP_RUNS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); +} + +#[cfg(test)] +pub(crate) fn reset_sweep_runs() { + SWEEP_RUNS.store(0, std::sync::atomic::Ordering::Relaxed); +} + +#[cfg(test)] +pub(crate) fn sweep_runs() -> usize { + SWEEP_RUNS.load(std::sync::atomic::Ordering::Relaxed) +} + +/// Serializes the tests that read [`sweep_runs`], since the counter is process-wide. +#[cfg(test)] +pub(crate) fn sweep_run_lock() -> &'static tokio::sync::Mutex<()> { + static LOCK: std::sync::OnceLock> = std::sync::OnceLock::new(); + LOCK.get_or_init(|| tokio::sync::Mutex::new(())) +} + // Test-only cost-gate counter (R8, U7): how many times the opportunistic repair // read an object's bytes on the skip path. The codec gate must spare a CIDv1/raw // row this read; the counter is the both-ways guard (removing the gate reads the @@ -1801,6 +1913,23 @@ mod tests { // `&Db` over a `PgPool`, so a failing-first wrapper cannot slot in without // changing signatures — see U6 seam note). + /// The re-arm intervals are expressed as multiples of the base so a test can shrink + /// all three coherently. This pins what they come to in production, which is the + /// number the constants' docs quote. + #[test] + fn the_rearm_multipliers_give_the_documented_production_intervals() { + assert_eq!( + SWEEP_REARM_DELAY.saturating_mul(SWEEP_IDLE_REARM_MULTIPLIER), + Duration::from_secs(3600), + "a run that repairs nothing waits an hour" + ); + assert_eq!( + SWEEP_REARM_DELAY.saturating_mul(SWEEP_FAILURE_REARM_MULTIPLIER), + Duration::from_secs(1800), + "a failed pass waits half an hour" + ); + } + #[tokio::test] async fn retry_lands_after_transient_failures() { let calls = Cell::new(0u32); diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 81db3e99..a3051860 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -433,7 +433,7 @@ async fn main() -> Result<()> { // Operator-tunable via GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS, read through the same // helper shape as the probe budget so the knob cannot be a silent no-op. ipfs_max_legacy_scan_rows: AppState::ipfs_legacy_scan_row_budget(&config), - ipfs_max_legacy_scan_rules: crate::api::ipfs::MAX_LEGACY_SCAN_RULES_PER_REQUEST, + ipfs_max_legacy_scan_rule_bytes: crate::api::ipfs::MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST, ipfs_scan_token_key: Arc::new(AppState::new_scan_token_key()), ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust, @@ -713,8 +713,9 @@ async fn main() -> Result<()> { /// need a walk. DETACHED, never on the boot path: the caller keeps serving while this /// runs, and the sweep's own batch bound plus inter-batch delay keep it off the DB's /// critical path. Its cursor is durable, so a restart mid-walk resumes instead of -/// rewinding. It re-arms on `SWEEP_REARM_DELAY` and so returns only on a failing pass -/// query, which is why the awaited value is no longer worth logging here. +/// rewinding. It re-arms after every run, including a failed one, so it never returns +/// and there is no awaited value to log here; the shutdown watcher below is what ends +/// it. /// /// A named function rather than an inline block in `main` so the WIRING has a seam a /// test can call: that the task is spawned at all, that it reads its batch and delay diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 7fbc6613..d4cf38ca 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -114,20 +114,22 @@ pub struct AppState { /// all-denying inventory paged the whole repo table at zero probes (#173 round 13, F2). /// Truncating here sheds a retryable 503 carrying a sealed continuation token. pub ipfs_max_legacy_scan_rows: usize, - /// Per-request ceiling on how many visibility RULES the CID resolver's legacy scan - /// may retain (default `api::ipfs::MAX_LEGACY_SCAN_RULES_PER_REQUEST`). The row - /// ceiling above bounds the row count but not the memory each row drags in: the - /// pager keeps every fetched page's rules for the whole request. Deliberately NOT an - /// operator knob (it is a memory guard, not a reach tradeoff); a field only for the - /// same test-seam reason as the sibling caps. - pub ipfs_max_legacy_scan_rules: usize, + /// Per-request ceiling on the BYTES of visibility rules the CID resolver's legacy + /// scan may retain (default `api::ipfs::MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST`). The + /// row ceiling above bounds the row count but not the memory each row drags in: the + /// pager keeps every fetched page's rules for the whole request, and neither the + /// number of rules per repo nor the length of a rule's reader list is capped, so a + /// rule COUNT would be the wrong unit. Deliberately NOT an operator knob (it is a + /// memory guard, not a reach tradeoff); a field only for the same test-seam reason as + /// the sibling caps. + pub ipfs_max_legacy_scan_rule_bytes: usize, /// Per-boot key sealing the legacy scan's continuation tokens (INV-13). /// /// The token is minted from a FETCHED row on a scan that served nothing, so by /// construction that row is a private or quarantined repo the caller may not read: /// its `created_at` and its `id` (which carries the owner's DID) are withheld /// fields. The token is therefore AEAD-SEALED, never signed plaintext and never - /// base64-of-plaintext — integrity is not confidentiality. Random per boot rather + /// base64-of-plaintext, since integrity is not confidentiality. Random per boot rather /// than derived or persisted: a scan continuation has no cross-restart meaning (a /// stale token simply fails to open and the caller restarts at the front, which is /// the same uniform absent behaviour a tampered token gets), and a per-boot key diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 6770a02a..468e796d 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -84,7 +84,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, ipfs_max_legacy_scan_rows: crate::api::ipfs::MAX_LEGACY_SCAN_ROWS_PER_REQUEST, - ipfs_max_legacy_scan_rules: crate::api::ipfs::MAX_LEGACY_SCAN_RULES_PER_REQUEST, + ipfs_max_legacy_scan_rule_bytes: crate::api::ipfs::MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST, ipfs_scan_token_key: Arc::new(crate::state::AppState::new_scan_token_key()), ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, @@ -6522,6 +6522,210 @@ mod tests { assert!(body.contains("public bytes"), "the object's bytes serve"); } + /// One transient database fault must not permanently disable the sweep. + /// + /// The wrapper was made periodic so coverage is wall-clock rather than a reboot + /// count. Returning for good on the first failed pass query undoes exactly that: a + /// single deadlock or connection reset disables legacy-CID repair for the whole + /// process lifetime, `main` never joins the handle, so nothing observes it past one + /// warn, and the node keeps withholding every unrepaired row until someone reboots + /// it. + /// + /// The fixture renames `pinned_cids` out of the way so every pass query fails, waits + /// for the loop to have gone round more than once (which a terminal return cannot + /// do), then renames the table back and asserts the still-running loop picks the + /// repair up. + /// + /// MUTATION (RED): restore the terminal `return` on `PassFailed` and the loop exits + /// on the first failure, so the row is never repaired. + #[sqlx::test] + async fn sweep_rearms_after_a_failed_pass(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let _serialized = crate::ipfs_pin::sweep_run_lock().lock().await; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["rearmsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("rearmsrc.git"); + let repo = seed_repo(&owner_did, "rearmsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let (raw_cid, _provider) = + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + // Every pass query now fails, exactly as a broken database makes them fail. + sqlx::query("ALTER TABLE pinned_cids RENAME TO pinned_cids_hidden") + .execute(&pool) + .await + .unwrap(); + + crate::ipfs_pin::reset_sweep_runs(); + let db = state.db.clone(); + let git_bin = state.git_bin.clone(); + // Short rather than literally zero: the loop is spinning against a real + // Postgres, and the property under test is that it goes round again at all. The + // failure and idle intervals are multiples of this base, so they shrink with it. + let handle = tokio::spawn(async move { + crate::ipfs_pin::run_sweep_rearmed( + std::path::Path::new("/tmp"), + &git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + std::time::Duration::from_millis(10), + &db, + ) + .await + }); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while crate::ipfs_pin::sweep_runs() < 2 && std::time::Instant::now() < deadline { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + assert!( + crate::ipfs_pin::sweep_runs() >= 2, + "a failed pass must re-arm: the sweep completed {} run(s) and stopped, which \ + is one transient database fault disabling legacy-CID repair for the life of \ + the process", + crate::ipfs_pin::sweep_runs() + ); + assert!( + !handle.is_finished(), + "the re-arm loop must never return; shutdown preempts it from the outside" + ); + + // The database comes back. The loop is still there to notice. + sqlx::query("ALTER TABLE pinned_cids_hidden RENAME TO pinned_cids") + .execute(&pool) + .await + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let mut repaired = false; + while std::time::Instant::now() < deadline { + if stored_pin(&pool, &fx.public_oid).await.0 == raw_cid { + repaired = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + handle.abort(); + assert!( + repaired, + "once the database recovers the still-running sweep must repair the row; a \ + wrapper that returned on the first failure never gets here" + ); + } + + /// A run that repairs nothing backs off; a run that repairs keeps the base interval. + /// + /// The base interval is priced against a settled table. It is not priced against the + /// table that never settles: source-less rows whose bytes are permanently gone cost + /// up to `MAX_DEAD_ROW_READS_PER_RUN` object reads per run and repair nothing, every + /// base interval, forever. Backing off on a fruitless run is what stops paying that; + /// resetting on a productive one is what keeps a table that is still yielding + /// repairs being walked often. + /// + /// Both directions, on the wall clock, off the run counter the wrapper exposes: + /// leg 1 is an empty table, where a run repairs nothing and the next run must NOT + /// arrive within a window several base intervals wide; leg 2 seeds a repairable row, + /// so the first run repairs and the second must arrive one BASE interval later. + /// + /// MUTATION (RED): drop the idle branch and leg 1 completes many runs in its window. + #[sqlx::test] + async fn sweep_backs_off_after_a_run_that_repairs_nothing(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let _serialized = crate::ipfs_pin::sweep_run_lock().lock().await; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + // Scaled down from production by a constant factor: the idle interval is a + // multiple of this base, so the ratio under test is the production ratio. + let base = std::time::Duration::from_millis(200); + let window = std::time::Duration::from_millis(800); + + let spawn_loop = |db: std::sync::Arc, git_bin: String| { + tokio::spawn(async move { + crate::ipfs_pin::run_sweep_rearmed( + std::path::Path::new("/tmp"), + &git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + base, + &db, + ) + .await + }) + }; + let await_first_run = || async { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while crate::ipfs_pin::sweep_runs() < 1 && std::time::Instant::now() < deadline { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + crate::ipfs_pin::sweep_runs() >= 1, + "fixture precondition: the sweep completes a first run" + ); + }; + + // Leg 1: nothing to repair. The next run must not arrive inside a window four + // base intervals wide. + crate::ipfs_pin::reset_sweep_runs(); + let idle_loop = spawn_loop(state.db.clone(), state.git_bin.clone()); + await_first_run().await; + tokio::time::sleep(window).await; + let idle_runs = crate::ipfs_pin::sweep_runs(); + idle_loop.abort(); + assert_eq!( + idle_runs, + 1, + "a run that repaired nothing must back off to the longer idle interval; at \ + the base interval this window fits about {} runs, each of which pays up to \ + MAX_DEAD_ROW_READS_PER_RUN fruitless object reads against a table that will \ + never repair", + window.as_millis() / base.as_millis() + ); + + // Leg 2: a repairable row. The run that repairs it must be followed by the BASE + // interval, so a second run lands well inside the same window. + let fx = seed_cid_repos(&slug, &short, &["idlesrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("idlesrc.git"); + let repo = seed_repo(&owner_did, "idlesrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let (raw_cid, _provider) = + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + crate::ipfs_pin::reset_sweep_runs(); + let busy_loop = spawn_loop(state.db.clone(), state.git_bin.clone()); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while crate::ipfs_pin::sweep_runs() < 2 && std::time::Instant::now() < deadline { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + let busy_runs = crate::ipfs_pin::sweep_runs(); + busy_loop.abort(); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "fixture precondition: the first run repairs the row" + ); + assert!( + busy_runs >= 2, + "a run that repaired something must keep the BASE interval; backing off \ + after a productive run would stall a table that is still yielding repairs \ + (saw {busy_runs} run(s))" + ); + } + /// U4 scenario 2 (#173): a legacy row whose object bytes are gone is left exactly /// as it is by the sweep: never rewritten, never deleted. The row stays withheld /// until the bytes come back, which is the non-destructive contract the skip-branch @@ -9328,46 +9532,6 @@ mod tests { ); } - /// F5 scenario 11 (#173 round 13): a failing pass query EXITS the wrapper. - /// - /// The re-arm loop is for a healthy node making slow progress. A broken database is - /// not that, and retrying it every re-arm would turn one logged failure into an - /// endless stream of them. The one-shot behavior a failing pass had before the - /// wrapper existed is preserved exactly. - #[sqlx::test] - async fn sweep_rearm_exits_on_pass_failure(pool: PgPool) { - let state = test_state(pool.clone()).await; - sqlx::query("DROP TABLE pinned_cids") - .execute(&pool) - .await - .unwrap(); - - let stats = tokio::time::timeout( - std::time::Duration::from_secs(30), - crate::ipfs_pin::run_sweep_rearmed( - std::path::Path::new("/tmp"), - &state.git_bin, - std::time::Duration::from_secs(5), - 16, - std::time::Duration::ZERO, - std::time::Duration::ZERO, - &state.db, - ), - ) - .await - .expect("a failing pass query must END the wrapper, never re-arm it forever"); - - assert_eq!( - stats.stop, - crate::ipfs_pin::SweepStop::PassFailed, - "the wrapper reports the failure it exited on" - ); - assert_eq!( - stats.repaired, 0, - "nothing was repaired against a broken table" - ); - } - /// F1 scenario 6 (#173, the collision case): two warm repos hold identical bytes, /// which is the shape (forks, a shared LICENSE blob, the empty tree) that makes an /// exclusive first-pinner claim wrong. Discovery records ONE additive source and @@ -10815,7 +10979,7 @@ mod tests { let mut state = test_state(pool).await; // Budget = one full scan of the single seeded repo: 1 page + 1 probe. The page // is charged because the legacy scan's DB-facing pages draw on this same bucket - // (#173 round 13, F2) — without that charge a denial-only inventory could be + // (#173 round 13, F2). Without that charge a denial-only inventory could be // re-paged for free by re-requesting. Production never sees a bucket this small: // `AppState::ipfs_work_budget` floors it at probes + pages, so only a fixture // that sets the limiter by hand has to do the arithmetic itself. @@ -10884,7 +11048,7 @@ mod tests { // pages draw on this same bucket (#173 round 13, F2), so re-requesting cannot // buy the inventory again for free; all four repos fit in one 128-row page, so // one page covers the whole scan. Production is floored at probes + pages by - // `AppState::ipfs_work_budget` — only a hand-set limiter does this arithmetic. + // `AppState::ipfs_work_budget`; only a hand-set limiter does this arithmetic. state.ipfs_work_rate_limiter = crate::rate_limit::RateLimiter::new(5, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; From e6cfb588c2a1600d59f2af88471d248a99532f03 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:37:57 -0500 Subject: [PATCH 55/77] fix(review): bound token key, rule bytes, and the discovery load An external review found three costs that outlive the request or the boot that paid for them. The scan token was sealed with a key minted per boot, so a restart or a rolling deploy made every outstanding continuation undecryptable and the handler reads an unopenable token as absent. A caller walking a large inventory could be sent back to the front indefinitely. The key is now derived from the node's Ed25519 seed with HKDF, so it is the same after a reload, under a distinct salt and info string so it is not the signing key and cannot be turned against signatures. Rotation is a version bump in the info string. An earlier comment rejected derivation on the grounds that surviving restarts bought nothing; that is exactly what it buys, and the comment now says so. The rule byte ceiling was summed after the page had already been fetched, so it truncated the request without bounding the work: the allocation and the transfer had happened. The budget now bounds the query itself. The cut lands on a repo boundary, which is not cosmetic: a mid-repo cut leaves a partial rule set, and a partial rule set at the gate is indistinguishable from a repo with no rules, so it would fail open and serve an object those rules withheld. Only fully loaded repos are gated. The query always admits the first rule-carrying repo of a page, so the bound is the budget plus one repo rather than the budget; without that a single oversized repo would put the cut at the cursor and every retry would reproduce the same page. The old check carried a not-exhausted condition so a short final page could not turn a complete scan into a permanent shed. That condition is gone because the taint no longer keys on page length; it keys on the query having left repos unloaded. A short final page that fits produces no cut and still 404s, and one that is cut is honestly incomplete and resumable. Discovery paged the whole repo table and stat'd every warm repo to pick a sixteen candidate window. Its comment called that acceptable because the cost was paid once, which was true when the sweep ran once per boot. This round made the sweep periodic, so an upgraded node with one unrepairable row paid a full table scan on every run, forever. The load now stops once the window is full. It collects one candidate past the cap so that running out still distinguishes a warm set that fits from one that does not, which is what the reset arm needs now that the total count is no longer known. Ordering is unchanged, so an attacker who mints repos still cannot move the boundary. Verified by mutation: a per-boot key strands a continuation across a restart, using the signing seed directly defeats the domain separation, summing bytes after the fetch materializes the whole page, paging to exhaustion buys pages the window cannot use, and reading the fit under cap from the window length loses the reset. --- crates/gitlawb-node/src/api/ipfs.rs | 289 +++++++++++++++++++++--- crates/gitlawb-node/src/auth/mod.rs | 3 +- crates/gitlawb-node/src/db/mod.rs | 107 +++++++++ crates/gitlawb-node/src/ipfs_pin.rs | 264 +++++++++++++++------- crates/gitlawb-node/src/main.rs | 5 +- crates/gitlawb-node/src/state.rs | 158 ++++++++++++- crates/gitlawb-node/src/test_support.rs | 93 +++++++- 7 files changed, 797 insertions(+), 122 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 8fc48ad8..c73481ef 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -132,9 +132,15 @@ pub(crate) const MAX_LEGACY_SCAN_ROWS_PER_REQUEST: usize = 2048; /// /// Bytes, not a rule count, because a count is the wrong unit for a memory bound: an /// owner controls how many rules their repos carry AND how long each rule's -/// `reader_dids` list is, so a handful of rules can retain as much as thousands. The -/// check runs inside `fetch_next_page` immediately after the rules query returns, so the -/// page that blows the budget truncates the request that bought it. +/// `reader_dids` list is, so a handful of rules can retain as much as thousands. +/// +/// Enforced IN THE QUERY (`Db::list_visibility_rules_for_repos_bounded`), not by summing +/// the page once it has landed. A post-fetch sum truncates the request but leaves the +/// transfer and the allocation already paid, so it bounds the result and not the work, +/// which is the wrong half of INV-10 on an anonymously reachable route. The query cuts on +/// a repo boundary and reports where; `fetch_next_page` drops the page's tail there and +/// mints a continuation, so the page that would have blown the budget truncates the +/// request that bought it without ever being materialized. /// /// Not an operator knob: it is a memory guard, not a reach/coverage tradeoff. 4 MiB is /// about 2 KiB per row at the default row ceiling, which is a generous rule set per repo @@ -191,11 +197,12 @@ struct LegacyScanPager { /// Bytes of visibility rules retained this request, the quantity the rules ceiling /// bounds. fetched_rule_bytes: usize, - /// Set by `fetch_next_page` when the page it just retained put - /// `fetched_rule_bytes` over the ceiling. The flag exists because the check has to - /// happen where the retention happens: measuring only when another page is - /// contemplated lets the page that actually blew the budget go unnoticed on a scan - /// that ends there. + /// Set by `fetch_next_page` when the rules query CUT the page it just fetched, that + /// is when the byte budget stopped the query part-way through the page's repos. The + /// flag exists because the decision has to happen where the fetch happens: measuring + /// only when another page is contemplated lets the page that actually blew the budget + /// go unnoticed on a scan that ends there, and measuring after the fetch bounds the + /// result rather than the work. rule_bytes_exceeded: bool, } @@ -263,18 +270,28 @@ impl LegacyScanPager { #[cfg(test)] note_scan_rows(page.len()); self.fetched_rows += page.len(); + // Measured on the FULL page the query returned, before any rules cut shortens it: + // this is the DB-facing row cost the row ceiling bounds, and a page that is short + // is a page with nothing behind it whatever the rules do. if page.len() < state.ipfs_legacy_scan_page_rows { self.exhausted = true; } if page.is_empty() { return Ok(()); } - let last = page.last().expect("non-empty page has a last row"); - self.cursor = Some((last.created_at_key.clone(), last.repo.id.clone())); + let mut page = page; let repo_ids: Vec = page.iter().map(|r| r.repo.id.clone()).collect(); - let rules = match tokio::time::timeout( + // The budget is per REQUEST, so what this page may spend is what is left of it. + // A cut ends the scan, so the remaining budget is only ever zero on a page bought + // after the always-admit escape overshot, and zero still admits one repo. + let budget_left = state + .ipfs_max_legacy_scan_rule_bytes + .saturating_sub(self.fetched_rule_bytes); + let (rules, cut_at) = match tokio::time::timeout( request_deadline.saturating_duration_since(std::time::Instant::now()), - state.db.list_visibility_rules_for_repos(&repo_ids), + state + .db + .list_visibility_rules_for_repos_bounded(&repo_ids, budget_left), ) .await { @@ -289,24 +306,42 @@ impl LegacyScanPager { return Err(budget_shed()); } }; - // Measure the page HERE, where it is retained, not on the next trip round the - // caller's loop. A rules query answers with whatever the matched repos carry; - // nothing bounds that per repo, so one page can be arbitrarily large and a check - // that only runs when another page is contemplated never sees it on a scan that - // ends on this one. Bytes rather than a rule count for the same reason: the - // quantity that grows is the length of each `reader_dids` list, not the number - // of rows in `visibility_rules`. + #[cfg(test)] + note_scan_rule_rows(rules.values().map(Vec::len).sum()); + // The bound lives in the QUERY, not in a sum taken once the page has landed. A + // rules query answers with whatever the matched repos carry, and nothing caps + // that per repo: a post-fetch sum truncated the request but left the transfer and + // the allocation already paid, which bounds the RESULT rather than the WORK. So + // the cut comes back from the database and the oversized tail is never + // materialized at all. Bytes rather than a rule count for the same reason as + // before: the quantity an owner can grow is the length of each `reader_dids` + // list, not the number of rows in `visibility_rules`. + if let Some(cut) = cut_at { + // The rows from the cut onward were never rule-loaded. Gating them against an + // empty rule map would read as "no restrictions" and FAIL OPEN, so they are + // dropped from the page entirely and the cursor stops in front of them. + // + // `max(1)` is belt and braces over the query's own guarantee that the first + // rule-carrying repo is always admitted. A cut at 0 would leave the cursor + // where it was, the caller's next request would reproduce this page exactly, + // and the ladder would be wedged on a permanent 503. + page.truncate(cut.max(1)); + self.rule_bytes_exceeded = true; + // This page had rows behind the cut, so the table is NOT covered even if the + // page itself was short. This replaces the old `!exhausted` condition: the + // taint now keys on the query having left repos unloaded rather than on the + // page's length. A short final page whose rules all fit produces no cut, so a + // scan that genuinely covered the table is still the definitive 404 it was, + // and a short final page that IS cut is honestly incomplete and resumable. + self.exhausted = false; + } + let last = page.last().expect("the cut always leaves at least one row"); + self.cursor = Some((last.created_at_key.clone(), last.repo.id.clone())); self.fetched_rule_bytes += rules .values() .flat_map(|v| v.iter()) .map(rule_retained_bytes) .sum::(); - // Only a page with more behind it truncates. A short page has already retained - // everything there is, so stopping on it would turn a scan that genuinely - // covered the table into a permanent 503 for an object that is simply absent. - if !self.exhausted && self.fetched_rule_bytes >= state.ipfs_max_legacy_scan_rule_bytes { - self.rule_bytes_exceeded = true; - } self.rules.extend(rules); self.rows.extend(page); Ok(()) @@ -317,7 +352,8 @@ impl LegacyScanPager { #[derive(serde::Deserialize)] pub struct ScanQuery { /// Sealed continuation from a previous truncated scan's 503 body. Opened with the - /// node's per-boot key and the request's canonical CID as associated data; ANY + /// key derived from the node's persistent identity (`AppState::derive_scan_token_key`, + /// so a restart does not invalidate it) and the request's canonical CID as associated data; ANY /// failure (undecryptable, tampered, expired, malformed, minted for another CID) is /// treated as absent and the scan starts at the front, identically and silently, so /// the token is no oracle. @@ -919,9 +955,9 @@ pub async fn get_by_cid( } // Rule-bytes ceiling: the row ceiling bounds rows, not the rules each // row drags in, and the pager retains every fetched page's rules for - // the whole request. The decision itself was taken inside - // `fetch_next_page`, against the page that did the retaining, so the - // request that bought an oversized page is the one that truncates. + // the whole request. The cut is made by the QUERY, so the oversized + // tail is never materialized; `fetch_next_page` drops the rows behind + // it and the request that asked for them is the one that truncates. if pager.rule_bytes_exceeded { walk.taint("rules-ceiling"); scan_continuation = pager.cursor.clone(); @@ -1842,6 +1878,31 @@ fn note_scan_rows(n: usize) { SCAN_ROWS.with(|c| c.set(c.get() + n)); } +// Test-only INV-10 cost counter: how many visibility-rule ROWS the legacy scan actually +// pulled out of the database this request. The byte ceiling is the guard, but a byte +// count computed from the rows AFTER they arrive cannot tell a bounded query from an +// unbounded one -- both report the same total. Counting the rows the query returned is +// what goes red when the bound moves back out of the query and into a post-fetch sum. +#[cfg(test)] +thread_local! { + static SCAN_RULE_ROWS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_scan_rule_rows() { + SCAN_RULE_ROWS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn scan_rule_rows() -> usize { + SCAN_RULE_ROWS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_scan_rule_rows(n: usize) { + SCAN_RULE_ROWS.with(|c| c.set(c.get() + n)); +} + // Test-only cost counter (F5, #173 round 11): how many times the fallback gate ran the // `pin_sources_at_cap` / `pin_sources_incomplete` pair. The work-budget peek sits ahead // of them, so an already-throttled caller leaves this at 0; putting the peek back after @@ -4506,6 +4567,176 @@ mod tests { ); } + /// The rule-bytes ceiling must be enforced by the QUERY, not by summing the page + /// after it has been transferred and allocated. + /// + /// A repo owner controls how many rules their repos carry and how long each + /// `reader_dids` list is, so a post-fetch sum truncates the REQUEST while leaving the + /// WORK unbounded: the oversized page is already in memory by the time the guard + /// fires. INV-10 bounds work done, never results measured afterwards, and the caller + /// here is an anonymous `/ipfs/{legacy-cid}` request holding one of the scarce walk + /// permits. + /// + /// The assertion is on the number of rule ROWS the query actually returned, not on + /// the status: the status is identical either way, which is exactly why the old shape + /// looked correct. + /// + /// MUTATION (RED): drop the query bound and sum the rules after the fetch, and the + /// whole page's rules are materialized (16 rows here against a budget that admits + /// one repo's two). + #[sqlx::test] + async fn get_by_cid_rule_bytes_bounded_in_the_query_not_after_the_page(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // One page holds every seeded repo, so nothing but the rule budget can bind. + state.ipfs_legacy_scan_page_rows = 8; + state.ipfs_max_legacy_scan_rows = 1000; + // Under this budget a single repo's pair of rules is already over, so at most one + // repo may be loaded and the page's remaining seven must never leave the database. + state.ipfs_max_legacy_scan_rule_bytes = 200; + seed_root_denying_repos(&state, "querybound", 8, 2).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let peer: SocketAddr = "203.0.113.171:5000".parse().unwrap(); + crate::api::ipfs::reset_scan_rule_rows(); + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + + let rule_rows = crate::api::ipfs::scan_rule_rows(); + assert!( + rule_rows <= 4, + "the byte budget must bound the QUERY: at most one repo's rules may be \ + materialized under a 200-byte budget, but {rule_rows} rule rows were pulled \ + (the whole page is 16)" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "reaching the query bound is the ceiling condition and sheds the retryable \ + 503, exactly as the post-fetch sum did: {body}" + ); + assert!( + body["message"] + .as_str() + .unwrap_or_default() + .contains("rules-ceiling"), + "the shed still names the rule-bytes ceiling: {body}" + ); + assert!( + continuation_of(&body).is_some(), + "and it still mints a continuation, or the repos behind the cut are \ + unreachable: {body}" + ); + } + + /// The property the old `!exhausted` condition protected, restated for the query + /// bound: a scan that genuinely covered the table must answer 404, never a permanent + /// 503. + /// + /// Under the query bound the taint no longer keys on "the page was short" but on + /// "the query left repos unloaded". A short final page whose rules all fit leaves + /// nothing unloaded, so it stays a complete scan and the absent object is a clean + /// 404. The budget here is finite and set by the fixture, so this is the guard being + /// exercised rather than the 4 MiB default never coming near. + #[sqlx::test] + async fn get_by_cid_short_final_page_under_the_rule_budget_still_404s(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 8; + state.ipfs_max_legacy_scan_rows = 1000; + // Roomy enough for all four repos' rules together, so no cut is possible. + state.ipfs_max_legacy_scan_rule_bytes = 64 * 1024; + seed_root_denying_repos(&state, "shortfit", 4, 1).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let peer: SocketAddr = "203.0.113.172:5000".parse().unwrap(); + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + + assert_eq!( + status, + StatusCode::NOT_FOUND, + "a complete scan of an absent object is a verdict; turning it into a 503 \ + would make the object permanently unresolvable: {body}" + ); + } + + /// The ladder MAKES PROGRESS under the query bound: every rung consumes at least one + /// repo, so the continuation always advances and the scan terminates. + /// + /// This is the failure mode the query bound could have introduced. If a page whose + /// FIRST repo alone exceeds the remaining budget loaded nothing, the cut would sit at + /// the cursor, the next request would reproduce it exactly, and the caller would be + /// wedged on a 503 forever for an object the node could otherwise settle. The bound + /// therefore always admits the first rule-carrying repo of a page whatever its size. + /// + /// The ladder ends on the tokenless shed, which is the design's "your ladder is + /// over" answer for a RESUMED scan (absence was only ever proven over + /// `[token, end)`), not on a 404. + #[sqlx::test] + async fn get_by_cid_rule_bytes_ladder_advances_to_a_tokenless_shed(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1000; + // Every repo's rules alone clear the budget, so every page is cut at its first + // repo: the worst case for progress. + state.ipfs_max_legacy_scan_rule_bytes = 1; + let repos = 8usize; + seed_root_denying_repos(&state, "ladder", repos, 2).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let peer: SocketAddr = "203.0.113.173:5000".parse().unwrap(); + let router = ipfs_router(state); + let mut token: Option = None; + let mut rungs = 0usize; + let bound = repos + 2; + loop { + rungs += 1; + assert!( + rungs <= bound, + "the ladder must consume at least one repo per rung and terminate within \ + {bound} rungs; a rung that loaded nothing would repeat forever" + ); + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::NOT_FOUND { + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "rung {rungs} must be the retryable 503: {body}" + ); + let next = continuation_of(&body); + if next.is_none() { + break; + } + assert_ne!( + next, token, + "rung {rungs} handed back the SAME continuation it was given, so the scan \ + made no progress and the caller is wedged: {body}" + ); + token = next; + } + } + /// Scenario 8: interleaved callers stay isolated. Two source keys alternate /// token-echoing ladders against the same denial-heavy inventory with the holder /// past the ceiling; each must reach its own 200 within its own bound. diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index b9e3f4b7..27b67786 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -490,6 +490,7 @@ mod tests { use clap::Parser; let keypair = Keypair::generate(); + let scan_token_key = crate::state::AppState::derive_scan_token_key(&keypair); let (ref_tx, _) = tokio::sync::broadcast::channel(1); let (task_tx, _) = tokio::sync::broadcast::channel(1); let pool = sqlx::postgres::PgPoolOptions::new() @@ -524,7 +525,7 @@ mod tests { ipfs_max_legacy_scan_rows: crate::api::ipfs::MAX_LEGACY_SCAN_ROWS_PER_REQUEST, ipfs_max_legacy_scan_rule_bytes: crate::api::ipfs::MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST, - ipfs_scan_token_key: Arc::new(crate::state::AppState::new_scan_token_key()), + ipfs_scan_token_key: Arc::new(scan_token_key), ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 1eb08e6b..0096cc75 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -4128,6 +4128,113 @@ impl Db { } Ok(out) } + + /// Visibility rules for one scan page, bounded IN THE QUERY by a byte budget. + /// + /// The unbounded sibling above is right for the listing surfaces: they read a page + /// the caller is already authorized for. It is wrong for the resolver's legacy scan, + /// which runs on an anonymously reachable route while holding scarce walk admission. + /// A repo owner controls both how many rules their repos carry and how long each + /// `reader_dids` list is, so summing the bytes AFTER the rows arrive truncates the + /// request without bounding the work: the oversized page has already been transferred + /// and allocated by the time the sum is taken (INV-10 bounds work done, never results + /// measured afterwards). + /// + /// The cut lands on a REPO boundary, never inside one. A partially loaded rule set is + /// indistinguishable at the gate from a repo with no rules at all, so a mid-repo cut + /// would FAIL OPEN and serve a path-scoped object the missing rules would have + /// denied. Every repo this returns is therefore complete, and the caller drops the + /// page's tail from the cut onward rather than gating it against rules it does not + /// have. + /// + /// `repo_ids` must be in the page's `(created_at, id)` order; the returned cut is the + /// 0-based index into that slice of the first repo whose rules did NOT fit, or `None` + /// when the whole page fit. Repos carrying no rules never cut. + /// + /// The FIRST rule-carrying repo of a page is admitted whatever its size, so a page + /// always makes progress. Without that a repo whose rules alone exceed the remaining + /// budget would put the cut at the cursor, the caller's next request would reproduce + /// it exactly, and the ladder would be wedged on a permanent 503. One repo's rule set + /// is the residual bound this leaves; the whole page's was the bound before. + pub async fn list_visibility_rules_for_repos_bounded( + &self, + repo_ids: &[String], + byte_budget: usize, + ) -> Result<( + std::collections::HashMap>, + Option, + )> { + use std::collections::HashMap; + if repo_ids.is_empty() { + return Ok((HashMap::new(), None)); + } + // `running` is a sum of non-negative per-repo sizes over the page order, so it is + // monotonic: once it passes the budget every later repo is excluded too, which is + // what makes "the kept set is a prefix" true and the single cut index meaningful. + // `rn = 1` is the always-admit escape for the first rule-carrying repo. + let rows = sqlx::query( + "WITH sized AS ( + SELECT v.id, v.repo_id, v.path_glob, v.mode, v.reader_dids, v.created_by, + v.created_at, + octet_length(v.id) + octet_length(v.repo_id) + + octet_length(v.path_glob) + octet_length(v.created_by) + + octet_length(v.reader_dids) AS b, + array_position($1::text[], v.repo_id) AS pos + FROM visibility_rules v + WHERE v.repo_id = ANY($1::text[]) + ), + per_repo AS ( + SELECT repo_id, pos, SUM(b) AS repo_bytes FROM sized GROUP BY repo_id, pos + ), + cum AS ( + SELECT repo_id, pos, + SUM(repo_bytes) OVER (ORDER BY pos ROWS UNBOUNDED PRECEDING) AS running, + ROW_NUMBER() OVER (ORDER BY pos) AS rn + FROM per_repo + ), + kept AS ( + SELECT repo_id, pos FROM cum WHERE running <= $2::bigint OR rn = 1 + ), + cut AS ( + SELECT MIN(pos) AS cut_pos FROM cum WHERE running > $2::bigint AND rn > 1 + ) + SELECT s.id, s.repo_id, s.path_glob, s.mode, s.reader_dids, s.created_by, + s.created_at, cut.cut_pos + FROM sized s + JOIN kept k ON k.repo_id = s.repo_id + CROSS JOIN cut + ORDER BY k.pos, s.path_glob", + ) + .bind(repo_ids) + .bind(byte_budget.min(i64::MAX as usize) as i64) + .fetch_all(&self.pool) + .await?; + + // `array_position` is 1-based and the caller indexes a slice. No rows means no + // rules matched the page at all, which is also no cut. + let cut_at = rows + .first() + .and_then(|r| r.get::, _>("cut_pos")) + .map(|pos| (pos as usize).saturating_sub(1)); + let mut out: HashMap> = HashMap::new(); + for r in rows { + let readers: String = r.get("reader_dids"); + let created_at: String = r.get("created_at"); + let rule = VisibilityRule { + id: r.get("id"), + repo_id: r.get("repo_id"), + path_glob: r.get("path_glob"), + mode: VisibilityMode::from_db(&r.get::("mode")), + reader_dids: serde_json::from_str(&readers).unwrap_or_default(), + created_by: r.get("created_by"), + created_at: created_at + .parse::>() + .unwrap_or_else(|_| Utc::now()), + }; + out.entry(rule.repo_id.clone()).or_default().push(rule); + } + Ok((out, cut_at)) + } } // ── Repo Stars ──────────────────────────────────────────────────────────────── diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 71843942..5d4579a3 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -27,6 +27,39 @@ use std::time::{Duration, Instant}; /// repos ONE background row may read. If either moves, the other does not follow. pub(crate) const MAX_LEGACY_DISCOVERY_PROBES: usize = 16; +// Test-only cost counters for the sweep's discovery load: how many keyset PAGES of +// `repos` one `load_discovery_ctx` bought, and how many ROWS they carried. A load that +// pages the table to exhaustion and one that stops as soon as the probe window is full +// are indistinguishable by outcome, so the window contents cannot go red on the +// difference; the paging cost is the only thing that can. +#[cfg(test)] +thread_local! { + static DISCOVERY_REPO_PAGES: std::cell::Cell = const { std::cell::Cell::new(0) }; + static DISCOVERY_REPO_ROWS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_discovery_paging() { + DISCOVERY_REPO_PAGES.with(|c| c.set(0)); + DISCOVERY_REPO_ROWS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn discovery_repo_pages() -> usize { + DISCOVERY_REPO_PAGES.with(|c| c.get()) +} + +#[cfg(test)] +pub(crate) fn discovery_repo_rows() -> usize { + DISCOVERY_REPO_ROWS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_discovery_page(rows: usize) { + DISCOVERY_REPO_PAGES.with(|c| c.set(c.get() + 1)); + DISCOVERY_REPO_ROWS.with(|c| c.set(c.get() + rows)); +} + /// Attempts (including the first) for a transient DB-record retry. const PIN_RECORD_ATTEMPTS: u32 = 3; /// Backoff between DB-record retry attempts. @@ -380,6 +413,14 @@ struct DiscoveryCtx { /// SQL order, so a key off by one character rotates the list to a boundary the query /// never had. candidates: Vec<(crate::db::RepoRecord, String, std::path::PathBuf)>, + /// Whether the node's WHOLE warm candidate set fits in one window, which is the + /// condition the traversal's continuation reset arm turns on. + /// + /// A separate field because `candidates` can no longer answer it. The load stops as + /// soon as the window is full, so `candidates.len()` is `MAX_LEGACY_DISCOVERY_PROBES` + /// on a node with seventeen warm repos and on a node with seventeen thousand alike. + /// `load_discovery_ctx` collects one candidate past the window purely to decide this. + warm_fits_under_cap: bool, /// The ceiling on the whole pass's discovery, so one pass costs at most one /// `git_timeout` in total on top of the per-row probe cap. Per PASS, not per run: /// `load_discovery_ctx` runs once per `sweep_pass` and a run loops passes. @@ -491,97 +532,162 @@ async fn load_discovery_ctx( git_timeout: Duration, db: &crate::db::Db, ) -> Result { - // Page to EXHAUSTION. The resolver's legacy scan drives this same query and - // deliberately does NOT (`api::ipfs`): it stops the moment its probe or visit budget - // is spent, because it runs on an anonymously reachable route while holding scarce - // walk admission, where reading the whole table is the amplification the budget - // exists to forbid. Same query, different threat model. This is background - // maintenance on a timer: no caller to amplify, no permit to pin, and the pass needs - // the whole warm candidate set before it can call a row settled. Do not "align" this - // loop with the resolver's: the budgets it stops on have no counterpart here. + // Paged only as far as the WINDOW needs, not to exhaustion. + // + // The exhaustive load was defended as "background maintenance on a timer" whose + // "paging cost is paid once", and that was true when the sweep ran once per boot: one + // full-table pass per process lifetime to choose sixteen candidates. The sweep now + // re-arms on a timer, so the cost is paid on every run for as long as the node holds a + // single unrepairable source-less row. The idle backoff stretches that to hourly; it + // does not bound it. Same query as the resolver's legacy scan and still a different + // threat model (no caller to amplify, no scarce permit pinned), but an unbounded read + // that repeats forever is worth stopping on its own account. // - // Per PASS, not per row: `load_discovery_ctx` already runs once per pass and its - // result is reused for every source-less row, so the paging cost is paid once. + // The window is unchanged. It is still the first `MAX_LEGACY_DISCOVERY_PROBES` WARM + // candidates strictly after the persisted continuation, wrapping to the front of the + // `(created_at, id)` order when the tail runs out, so the candidates picked here are + // byte for byte the ones the exhaustive load rotated to. What changed is that the + // rotation now STEERS the paging instead of being applied to a list already read: + // phase 0 reads forward from the continuation, phase 1 wraps to the front and stops + // where phase 0 began, and either may stop early once the window is full. + // + // Ordering is still the QUERY's `(created_at, id)` ASC, so non-steerability is + // untouched: `repo_id` derives from a grindable owner DID, but minted repos carry a + // fresh `created_at` and sort LAST, where they can only ever be reached after the + // older true holder rather than instead of it. + // + // Per PASS, not per row: `load_discovery_ctx` still runs once per pass and its result + // is still reused for every source-less row in that pass, so all of them share one + // window. let page_rows = crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS; - let mut cursor: Option<(String, String)> = None; - let mut candidates: Vec<(crate::db::RepoRecord, String)> = Vec::new(); - loop { - let page = db - .list_repos_page_for_scan( - cursor - .as_ref() - .map(|(created_at, id)| (created_at.as_str(), id.as_str())), - page_rows as i64, - ) - .await?; - let Some(last) = page.last() else { break }; - cursor = Some((last.created_at_key.clone(), last.repo.id.clone())); - let last_page = page.len() < page_rows; - candidates.extend( - page.into_iter() - .filter(|r| !r.quarantined) - .map(|r| (r.repo, r.created_at_key)), - ); - if last_page { - break; - } - } + + // Read BEFORE paging, because the load is steered by it now. + let (cont_created_at, cont_id) = db.discovery_continuation().await?; + let resumed = !cont_created_at.is_empty() || !cont_id.is_empty(); + + // ONE PAST the window. The exhaustive load could read "the whole warm list fits under + // the cap" off a total count it had in hand; a bounded load has no total. Collecting + // one extra candidate restores the decision without restoring the cost: a load that + // stops at `MAX + 1` has PROVEN there are more than `MAX` warm candidates, and a load + // that ends at `MAX` or fewer can only have done so by running the whole warm set to + // its end. So `warm.len() <= MAX` after the fact is exactly the old condition. + let want = MAX_LEGACY_DISCOVERY_PROBES + 1; let repos_dir = repos_dir.to_path_buf(); - let warm = tokio::task::spawn_blocking(move || { - candidates - .into_iter() - .filter_map(|(repo, created_at_key)| { - match crate::git::repo_store::validated_repo_disk_path( - &repos_dir, - &repo.owner_did, - &repo.name, - ) { - Ok(p) if p.is_dir() => Some((repo, created_at_key, p)), - // Cold: not on this node's disk right now. It is not evidence about - // any row (see `discover_legacy_row`), so it is simply absent here. - Ok(_) => None, - Err(e) => { - tracing::warn!(repo_id = %repo.id, err = %e, "sweep discovery: rejected unsafe repo path"); - None + let mut warm: Vec<(crate::db::RepoRecord, String, std::path::PathBuf)> = Vec::new(); + + for phase in 0..2 { + if warm.len() >= want { + break; + } + // Nothing persisted means phase 0 already started at the front, so there is no + // prefix left to wrap into. + if phase == 1 && !resumed { + break; + } + let mut cursor: Option<(String, String)> = if phase == 0 && resumed { + Some((cont_created_at.clone(), cont_id.clone())) + } else { + None + }; + // Phase 1 must not run past the point phase 0 started at, or the wrap would probe + // the same candidates twice and the window would be short by however many it + // repeated. + let stop_after: Option<(&str, &str)> = if phase == 1 { + Some((cont_created_at.as_str(), cont_id.as_str())) + } else { + None + }; + loop { + let need = want.saturating_sub(warm.len()); + if need == 0 { + break; + } + let page = db + .list_repos_page_for_scan( + cursor + .as_ref() + .map(|(created_at, id)| (created_at.as_str(), id.as_str())), + page_rows as i64, + ) + .await?; + #[cfg(test)] + note_discovery_page(page.len()); + let Some(last) = page.last() else { break }; + let last_page = page.len() < page_rows; + cursor = Some((last.created_at_key.clone(), last.repo.id.clone())); + let mut wrapped_to_start = false; + let mut candidates: Vec<(crate::db::RepoRecord, String)> = Vec::new(); + for r in page { + if let Some((created_at, id)) = stop_after { + if (r.created_at_key.as_str(), r.repo.id.as_str()) > (created_at, id) { + wrapped_to_start = true; + break; } } - }) - .collect::>() - }) - .await?; - let mut warm = warm; - - // ROTATE to the traversal's window. The list is already in `(created_at, id)` order, - // so the window is the first `MAX_LEGACY_DISCOVERY_PROBES` entries strictly after the - // persisted continuation, wrapping through the prefix when it runs off the end. - // - // After the warm filter, deliberately: a cold or quarantined candidate is not a - // window slot the traversal spent, so rotating first would let a node full of cold - // repos advance the continuation past warm candidates nobody ever probed. The window - // is sixteen WARM candidates. - // - // Every pass of a traversal reads the same persisted value (it only moves in the - // traversal-ending pass), so the window is stable across the traversal by - // construction and two source-less rows in different passes probe the same repos. - let (cont_created_at, cont_id) = db.discovery_continuation().await?; - if !cont_created_at.is_empty() || !cont_id.is_empty() { - let split = warm - .iter() - .position(|(repo, created_at_key, _)| { - (created_at_key.as_str(), repo.id.as_str()) - > (cont_created_at.as_str(), cont_id.as_str()) - }) - .unwrap_or(warm.len()); - warm.rotate_left(split); + // QUARANTINE, dropped before the stat so a hidden repo costs nothing. + if r.quarantined { + continue; + } + candidates.push((r.repo, r.created_at_key)); + } + warm.extend(warm_candidates(&repos_dir, candidates, need).await?); + if wrapped_to_start || last_page { + break; + } + } } + // The fit-under-cap arm the traversal's continuation reset depends on, decided from + // the one extra candidate rather than from a whole-table count (see `want`). + let warm_fits_under_cap = warm.len() <= MAX_LEGACY_DISCOVERY_PROBES; + warm.truncate(MAX_LEGACY_DISCOVERY_PROBES); + Ok(DiscoveryCtx { candidates: warm, + warm_fits_under_cap, pass_deadline: Instant::now() + git_timeout, }) } +/// Keep the WARM ones out of a batch of candidate rows, stopping after `need` of them. +/// +/// The stat runs on the blocking pool because it is O(rows) filesystem calls and would +/// otherwise park a tokio worker for the length of a sweep. `need` is what keeps a page's +/// tail from being stat'd once the window is already full: the caller stops paging at that +/// point, so those rows are never looked at again this pass either. +/// +/// An UNSAFE PATH is dropped with a warn and is terminal, and a COLD repo is simply +/// absent: neither is evidence about any row (see `discover_legacy_row`). +async fn warm_candidates( + repos_dir: &std::path::Path, + candidates: Vec<(crate::db::RepoRecord, String)>, + need: usize, +) -> Result> { + let repos_dir = repos_dir.to_path_buf(); + Ok(tokio::task::spawn_blocking(move || { + let mut out = Vec::new(); + for (repo, created_at_key) in candidates { + if out.len() >= need { + break; + } + match crate::git::repo_store::validated_repo_disk_path( + &repos_dir, + &repo.owner_did, + &repo.name, + ) { + Ok(p) if p.is_dir() => out.push((repo, created_at_key, p)), + Ok(_) => {} + Err(e) => { + tracing::warn!(repo_id = %repo.id, err = %e, "sweep discovery: rejected unsafe repo path"); + } + } + } + out + }) + .await?) +} + /// What discovery did with one source-less legacy row, in the same three-way shape /// [`RepairOutcome`] uses so the row accounting is unchanged. enum DiscoveryOutcome { @@ -667,7 +773,7 @@ async fn discover_legacy_row( // the whole warm list fits in one window. Together they pick the traversal's advance // arm once the row is done. let mut live_probes = 0usize; - let fits_under_cap = ctx.candidates.len() <= MAX_LEGACY_DISCOVERY_PROBES; + let fits_under_cap = ctx.warm_fits_under_cap; // Every candidate that gets this far is READ, so taking the first // MAX_LEGACY_DISCOVERY_PROBES bounds the expensive work exactly. Candidates the // filters already rejected never reach here and so cost nothing against the cap. @@ -751,7 +857,7 @@ async fn discover_legacy_row( } } traversal.note_row(live_probes, fits_under_cap); - if ctx.candidates.len() > MAX_LEGACY_DISCOVERY_PROBES { + if !ctx.warm_fits_under_cap { // Cap exhausted with candidates left unprobed: RETRYABLE, never terminal. The // probe order is deterministic, but "a re-walk finds the same nothing" only // holds if the candidate set cannot be steered, and it can: repo ids derive diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index a3051860..66bfa096 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -116,6 +116,9 @@ async fn main() -> Result<()> { // Load or generate the node's identity keypair let keypair = load_or_create_keypair(&config)?; + // Sealing key for the legacy-scan continuation tokens, DERIVED from the identity + // just loaded so it is the same key after a restart (see `derive_scan_token_key`). + let scan_token_key = AppState::derive_scan_token_key(&keypair); let node_did = keypair.did(); // One-time metrics init. Must run before any handler that calls into @@ -434,7 +437,7 @@ async fn main() -> Result<()> { // helper shape as the probe budget so the knob cannot be a silent no-op. ipfs_max_legacy_scan_rows: AppState::ipfs_legacy_scan_row_budget(&config), ipfs_max_legacy_scan_rule_bytes: crate::api::ipfs::MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST, - ipfs_scan_token_key: Arc::new(AppState::new_scan_token_key()), + ipfs_scan_token_key: Arc::new(scan_token_key), ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust, sync_trigger_rate_limiter, diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index d4cf38ca..85356fb4 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -8,6 +8,19 @@ use crate::git::repo_store::RepoStore; use crate::p2p::P2pHandle; use crate::rate_limit::RateLimiter; +/// HKDF salt for [`AppState::derive_scan_token_key`]. A constant, not a secret: HKDF's +/// salt is a domain qualifier, and the confidentiality of the derived key rests entirely +/// on the node's private seed being the input keying material. +const SCAN_TOKEN_KEY_SALT: &[u8] = b"gitlawb/hkdf-salt/ipfs-scan-token"; + +/// HKDF `info` for [`AppState::derive_scan_token_key`]: the domain separation AND the +/// rotation handle in one string. Nothing else in the node derives from this label, so +/// the token key is unrelated to the signing key it shares an input with; bumping the +/// trailing version rotates every node's token key on its next boot, which invalidates +/// outstanding continuations (they simply fail to open and the caller restarts at the +/// front) and needs no migration, no config, and no change to this function. +const SCAN_TOKEN_KEY_INFO: &[u8] = b"gitlawb/ipfs-scan-token/v1"; + #[derive(Clone, Debug)] pub struct RefUpdateBroadcast { pub repo: String, @@ -119,21 +132,38 @@ pub struct AppState { /// row ceiling above bounds the row count but not the memory each row drags in: the /// pager keeps every fetched page's rules for the whole request, and neither the /// number of rules per repo nor the length of a rule's reader list is capped, so a - /// rule COUNT would be the wrong unit. Deliberately NOT an operator knob (it is a + /// rule COUNT would be the wrong unit. Enforced by the rules QUERY + /// (`Db::list_visibility_rules_for_repos_bounded`) rather than by a sum taken once the + /// page has landed, so the oversized page is never materialized at all. Deliberately + /// NOT an operator knob (it is a /// memory guard, not a reach tradeoff); a field only for the same test-seam reason as /// the sibling caps. pub ipfs_max_legacy_scan_rule_bytes: usize, - /// Per-boot key sealing the legacy scan's continuation tokens (INV-13). + /// Key sealing the legacy scan's continuation tokens (INV-13), derived from the + /// node's persistent identity by [`AppState::derive_scan_token_key`]. /// /// The token is minted from a FETCHED row on a scan that served nothing, so by /// construction that row is a private or quarantined repo the caller may not read: /// its `created_at` and its `id` (which carries the owner's DID) are withheld /// fields. The token is therefore AEAD-SEALED, never signed plaintext and never - /// base64-of-plaintext, since integrity is not confidentiality. Random per boot rather - /// than derived or persisted: a scan continuation has no cross-restart meaning (a - /// stale token simply fails to open and the caller restarts at the front, which is - /// the same uniform absent behaviour a tampered token gets), and a per-boot key - /// bounds the window in which any single key seals anything. + /// base64-of-plaintext, since integrity is not confidentiality. + /// + /// DERIVED, not random per boot. This reverses an earlier revision of this design, + /// which argued that derivation "would make old tokens valid across restarts for no + /// benefit and tie a throwaway transport secret to a long-lived signing key." Both + /// halves were wrong. Surviving a restart IS the benefit: the token is the ONLY way + /// a caller resumes a ladder, an unopenable one is treated as absent, and a caller + /// treated as absent silently restarts at the front of the scan. A node whose + /// inventory needs several ladder steps and which deploys more often than a caller + /// can climb therefore keeps that caller from ever reaching a holder, which + /// contradicts the reach bound the README states. And the tie to the signing key is + /// what the HKDF domain separation removes: the derived key is a one-way function of + /// the seed under an `info` string nothing else uses, so it is not the signing key + /// and cannot be worked back into one. + /// + /// Persisting a random key instead would need a schema change for a value the node + /// already has on disk, so derivation is also the smaller mechanism. Rotation is a + /// bump of the version in [`SCAN_TOKEN_KEY_INFO`]. pub ipfs_scan_token_key: Arc<[u8; 32]>, /// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` will buffer and /// serve (default `api::ipfs::MAX_SERVED_OBJECT_BYTES`). The serve reads via a @@ -358,10 +388,41 @@ impl AppState { config.ipfs_max_legacy_scan_rows } - /// A fresh random key for sealing legacy-scan continuation tokens (INV-13). - /// Drawn from the OS CSPRNG at construction; see `ipfs_scan_token_key`. - pub(crate) fn new_scan_token_key() -> [u8; 32] { - gitlawb_core::scan_token::new_key() + /// The key sealing legacy-scan continuation tokens (INV-13), DERIVED from the node's + /// persistent identity rather than minted per boot. + /// + /// HKDF-SHA256 over the node's Ed25519 seed, the same key material + /// `load_or_create_keypair` reads back from its PKCS#8 PEM on every boot, so the + /// derived key is byte-identical after a restart or a rolling deploy. + /// + /// Two properties the derivation has to carry, both load-bearing: + /// + /// * DOMAIN SEPARATION. The `info` string below is unique to this use, so the + /// derived key is not the signing seed and is not any other secret derived from + /// it. HKDF is one-way, so a leaked token key yields nothing about the signing + /// key and cannot be turned against a signature. + /// * A VERSION component, carried in the same `info` string + /// ([`SCAN_TOKEN_KEY_INFO`]) alongside a fixed [`SCAN_TOKEN_KEY_SALT`]. Bumping + /// the version rotates every token key on the next boot without touching the + /// identity, the token format, or any caller, so rotation is a constant change + /// rather than a fork of this derivation. + pub(crate) fn derive_scan_token_key(keypair: &Keypair) -> [u8; 32] { + use hmac::{Hmac, Mac}; + type HmacSha256 = Hmac; + + let seed = keypair.to_seed(); + // HKDF-Extract: PRK = HMAC(salt, ikm). + let mut extract = + HmacSha256::new_from_slice(SCAN_TOKEN_KEY_SALT).expect("HMAC takes a key of any size"); + extract.update(seed.as_slice()); + let prk = extract.finalize().into_bytes(); + // HKDF-Expand, one block: T(1) = HMAC(PRK, info || 0x01). One 32-byte output + // needs exactly one block of SHA-256, so there is no counter loop to get wrong. + let mut expand = + HmacSha256::new_from_slice(prk.as_slice()).expect("HMAC takes a key of any size"); + expand.update(SCAN_TOKEN_KEY_INFO); + expand.update(&[0x01]); + expand.finalize().into_bytes().into() } /// Work-budget capacity for [`ipfs_work_rate_limiter`](Self#structfield.ipfs_work_rate_limiter) @@ -1369,3 +1430,78 @@ mod repo_write_lease_tests { } } } + +#[cfg(test)] +mod scan_token_key_tests { + use super::AppState; + use gitlawb_core::identity::Keypair; + use gitlawb_core::scan_token::{open_scan_token, seal_scan_token, ScanPosition}; + + const CID: &str = "bafkreiscantokenkeyfixtureaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + + fn pos() -> ScanPosition { + ScanPosition { + created_at_key: "2020-01-01T12:00:00+00:00".to_string(), + id: "did:key:z6MkScanTokenOwner/repo".to_string(), + } + } + + /// The RESTART case. A token minted before a restart must still open after it, or a + /// caller laddering a deep inventory is silently returned to the front of the scan on + /// every rolling deploy and can never reach a holder buried past one window. + /// + /// The second key is derived from the identity RELOADED THROUGH ITS ON-DISK PEM, + /// which is exactly what `load_or_create_keypair` does on boot, so this exercises the + /// real restart path rather than a clone of the in-memory keypair. + #[test] + fn scan_token_key_survives_a_restart_of_the_same_identity() { + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("the identity serializes"); + let reloaded = Keypair::from_pem(&pem).expect("the identity reloads"); + + let before = AppState::derive_scan_token_key(&kp); + let after = AppState::derive_scan_token_key(&reloaded); + + let token = seal_scan_token(&before, CID, &pos(), i64::MAX - 1).expect("seal"); + assert_eq!( + open_scan_token(&after, CID, &token, 0), + Some(pos()), + "a continuation minted before a restart must open after it: the node's \ + identity is the same, so the derived sealing key must be too" + ); + } + + /// The must-not: a DIFFERENT node identity must derive a DIFFERENT key, so a token is + /// no more portable between nodes than it was when the key was random per boot. + #[test] + fn scan_token_key_does_not_open_under_a_different_identity() { + let mine = Keypair::generate(); + let theirs = Keypair::generate(); + + let token = seal_scan_token( + &AppState::derive_scan_token_key(&mine), + CID, + &pos(), + i64::MAX - 1, + ) + .expect("seal"); + assert_eq!( + open_scan_token(&AppState::derive_scan_token_key(&theirs), CID, &token, 0), + None, + "a continuation sealed by one node must not open under another node's identity" + ); + } + + /// Domain separation, executed rather than asserted in prose: the derived token key + /// must not be the signing seed itself. Compromising a token key must not hand an + /// attacker the material that signs. + #[test] + fn scan_token_key_is_not_the_signing_seed() { + let kp = Keypair::generate(); + assert_ne!( + AppState::derive_scan_token_key(&kp), + *kp.to_seed(), + "the token key must be a DERIVED secret, never the Ed25519 signing seed" + ); + } +} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 468e796d..785ebf35 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -55,6 +55,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { use clap::Parser; let keypair = Keypair::generate(); + let scan_token_key = crate::state::AppState::derive_scan_token_key(&keypair); let node_did = keypair.did(); let (ref_tx, _) = tokio::sync::broadcast::channel(1); let (task_tx, _) = tokio::sync::broadcast::channel(1); @@ -85,7 +86,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, ipfs_max_legacy_scan_rows: crate::api::ipfs::MAX_LEGACY_SCAN_ROWS_PER_REQUEST, ipfs_max_legacy_scan_rule_bytes: crate::api::ipfs::MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST, - ipfs_scan_token_key: Arc::new(crate::state::AppState::new_scan_token_key()), + ipfs_scan_token_key: Arc::new(scan_token_key), ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), @@ -8673,6 +8674,96 @@ mod tests { } } + /// The discovery load is bounded to ONE probe window, not to the whole repo table. + /// + /// The exhaustive load was justified as "background maintenance on a timer" whose + /// "paging cost is paid once". That was written when the sweep ran once per boot. + /// The sweep now re-arms on a timer, so a node carrying a single unrepairable + /// source-less row paid a full-table paging pass plus a stat of every warm repo on + /// every re-armed run, forever, to choose sixteen candidates. The idle backoff makes + /// that hourly rather than every five minutes, which is a smaller bill for the same + /// unbounded work. + /// + /// The window itself is unchanged, which is why the assertion is on the PAGING and + /// not on the outcome: an exhaustive load and a bounded one pick the same sixteen + /// candidates and reach the same verdict, so nothing about the result can go red on + /// the difference. The fixture puts more than one window of warm candidates at the + /// front of the `(created_at, id)` order and enough cold rows behind them to push the + /// table past a single page. + /// + /// MUTATION (RED): page to exhaustion and the load buys a second page it has no use + /// for. + #[sqlx::test] + async fn sweep_discovery_load_stops_once_the_window_is_full(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + // The bytes live in a bare with no `repos` row, so no candidate ever holds them + // and the row stays source-less: the pass runs a full window of probes. + let fx = seed_cid_repos(&slug, &short, &["boundsrc"]); + let src = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("boundsrc.git"); + let _warm = seed_candidate_ladder( + &state.db, + &owner_did, + &slug, + "boundwarm", + crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES + 4, + None, + ) + .await; + // Cold rows: a `repos` row with nothing on disk. They cost a page each but can + // never fill a window slot, so they are what an exhaustive load pages through. + let page_rows = crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS; + for pos in 100..(100 + page_rows) { + let repo = seed_repo_at(&owner_did, &format!("boundcold{pos}"), pos as i64); + state.db.create_repo(&repo).await.expect("seed a cold row"); + } + seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; + + crate::ipfs_pin::reset_discovery_paging(); + let stats = tokio::time::timeout( + std::time::Duration::from_secs(120), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + &mut Default::default(), + ), + ) + .await + .expect("the traversal terminates"); + + let pages = crate::ipfs_pin::discovery_repo_pages(); + let rows = crate::ipfs_pin::discovery_repo_rows(); + assert_eq!( + pages, 1, + "the window fills inside the first page, so the load must stop there; it \ + bought {pages} pages carrying {rows} rows" + ); + assert!( + rows <= page_rows, + "a bounded load reads at most the pages it needs; it read {rows} rows out of \ + a table of {}", + crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES + 4 + page_rows + ); + assert_eq!( + stats.dead_row_reads, + crate::ipfs_pin::MAX_LEGACY_DISCOVERY_PROBES, + "and the window it picked is still a FULL one: bounding the load must not \ + shrink the number of candidates the row actually probes" + ); + } + /// F5 scenario 1 (#173 round 13): a holder past the probe cap is REACHED. /// /// `discover_legacy_row` probes the first `MAX_LEGACY_DISCOVERY_PROBES` of a list From 75c0e282af187f49025eed06e63befb574601cb1 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Thu, 13 Aug 2026 09:55:24 -0500 Subject: [PATCH 56/77] test(node): probe the production advisory-lock key, not a stale copy Both write-lock tests hand-copied advisory_lock_key's derivation. The copy still used DefaultHasher, so once the key moved to SHA-256 (#210) the probe checked a key nobody held. The disconnect test then failed asserting the lock was held, and the success test passed asserting a release it never actually observed. Import the production function, as the acquire-deadline test already does, so the two cannot diverge again. --- crates/gitlawb-node/src/api/repos.rs | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 98c4abdd..7b3676db 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -6146,17 +6146,6 @@ mod tests { ); } - /// Reproduce `repo_store::advisory_lock_key` (private there) so a test can probe the - /// exact key `acquire_write` derives. - #[cfg(unix)] - fn write_lock_key(owner_slug: &str, repo_name: &str) -> i64 { - use std::hash::{Hash, Hasher}; - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - owner_slug.hash(&mut hasher); - repo_name.hash(&mut hasher); - hasher.finish() as i64 - } - #[cfg(unix)] fn pid_alive(pid: i32) -> bool { // SAFETY: kill(2) with signal 0 only probes; it takes integers and borrows no @@ -6262,7 +6251,10 @@ mod tests { .unwrap(); // The mirror row stores the short owner as owner_did, so the slug is the owner. - let key = write_lock_key(&owner.replace([':', '/'], "_"), name); + // Import the production derivation rather than hand-copying it: a local copy + // silently diverged when the key moved to SHA-256 (#210). + use crate::git::repo_store::advisory_lock_key; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), name); // Probe from a pool that is NOT the store's lock pool and NOT the harness pool. let probe = sqlx::postgres::PgPoolOptions::new() .max_connections(2) @@ -6427,7 +6419,8 @@ mod tests { .await .unwrap(); - let key = write_lock_key(&owner.replace([':', '/'], "_"), name); + use crate::git::repo_store::advisory_lock_key; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), name); let probe = sqlx::postgres::PgPoolOptions::new() .max_connections(2) .connect_lazy_with((*pool.connect_options()).clone()); From a60026bbfa07353a1cf8aa207df4719f5f20821c Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:54:41 -0500 Subject: [PATCH 57/77] test(node): pin the legacy scan's DB-facing row selection to the configured ceiling The row ceiling is enforced in the scan loop, after a full page has already been fetched and rule-loaded, so a cap below the page size is ignored for the first fetch and any cap that is not a page multiple overshoots by up to a page. The existing coverage could not catch that. Both row-ceiling tests asserted rows <= ceiling + one page, which passes whether or not the query is bounded, and rows-returned cannot distinguish a bounded query from a full fetch that is trimmed afterwards. Add a scan_limit seam recording the limit actually sent to the database, tighten both tolerant assertions to exact counts, and add two tests that pin the limit: a ceiling below the page size and a ceiling that is not a page multiple. Also add the fail-open fixture. A repo evaluated with a partial rule set is served as if it had no rules, so the boundary repo is seeded public at root with a path-scoped rule and a real blob on disk, pinned with NULL provenance so the scan path's rule map is the one consulted, and asserted never served across its own ladder. Its positive control serves the same blob with the rule absent, so a failure there is attributable to the rules rather than to a probe or a budget skip. Both new tests fail against current code on the scan_limit assertion. --- crates/gitlawb-node/src/api/ipfs.rs | 387 +++++++++++++++++++++++++++- 1 file changed, 379 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index c73481ef..57cf8381 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -248,6 +248,11 @@ impl LegacyScanPager { .cursor .as_ref() .map(|(created_at, id)| (created_at.as_str(), id.as_str())); + // Record the DB-facing ask before it is made. It sits above the timeout opener + // because the INV-22 guard reads a fixed lookback from the query call for that + // wrapper, and anything inserted inside the window eats its margin. + #[cfg(test)] + note_scan_limit(state.ipfs_legacy_scan_page_rows); let page = match tokio::time::timeout( request_deadline.saturating_duration_since(std::time::Instant::now()), state @@ -1878,6 +1883,31 @@ fn note_scan_rows(n: usize) { SCAN_ROWS.with(|c| c.set(c.get() + n)); } +// Test-only counter for the LIMIT the legacy scan actually sends to SQL, summed over +// the request's fetches. The row counter above measures what came BACK, so it cannot +// tell a query that asked for the remaining budget from one that asked for a full page +// and then dropped the tail: both return the same rows. The limit is the DB-facing ask, +// which is the quantity the operator ceiling is supposed to bound. +#[cfg(test)] +thread_local! { + static SCAN_LIMIT: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_scan_limit() { + SCAN_LIMIT.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn scan_limit() -> usize { + SCAN_LIMIT.with(|c| c.get()) +} + +#[cfg(test)] +fn note_scan_limit(n: usize) { + SCAN_LIMIT.with(|c| c.set(c.get() + n)); +} + // Test-only INV-10 cost counter: how many visibility-rule ROWS the legacy scan actually // pulled out of the database this request. The byte ceiling is the guard, but a byte // count computed from the rows AFTER they arrive cannot tell a bounded query from an @@ -3767,11 +3797,10 @@ mod tests { // The row COUNT first: it is the cost this ceiling exists to bound, and a // status-first ordering would attribute a missing ceiling to the tail instead. let rows = crate::api::ipfs::scan_rows(); - assert!( - rows <= 4 + 2, - "the ceiling (4) bounds the DB-facing selection to at most one page (2) of \ - overshoot; a denial-only inventory must not page the whole table. Read {rows} \ - of 12 seeded rows" + assert_eq!( + rows, 4, + "the ceiling (4) bounds the DB-facing selection exactly; a denial-only \ + inventory must not page the whole table. Read {rows} of 12 seeded rows" ); assert_eq!( status, @@ -3842,10 +3871,10 @@ mod tests { ); assert_eq!(body["error"], "search_incomplete", "{body}"); let rows = crate::api::ipfs::scan_rows(); - assert!( - rows <= 4 + 2, + assert_eq!( + rows, 4, "quarantine costs neither a probe nor a visit, so only the ROW ceiling can \ - stop this pager. Read {rows} of 12 seeded rows" + stop this pager, and it stops it exactly. Read {rows} of 12 seeded rows" ); assert!( continuation_of(&body).is_some(), @@ -4109,6 +4138,348 @@ mod tests { ); } + /// Scenario 7 (#173 round 14, F4): an operator ceiling BELOW the page size must + /// bound the QUERY, not just the loop that reads its result. + /// + /// Page size 4, ceiling 2. The scan may prove two rows, so two rows is what it may + /// select and rule-load inside the admission-held, budget-clamped region. A fetch + /// that always asks for a full page buys twice the ceiling and the row arm only + /// notices afterwards, which makes the page size an implicit floor under the knob. + /// `scan_limit()` is what separates the fix from a post-fetch trim: it records the + /// DB-facing ask, so a trim that hands back the same two rows still reads 4 here. + /// + /// The fixture also carries the fail-open boundary case. The LAST row is a real bare + /// repo, public at "/", holding a real blob at /src/secret.txt behind a path-scoped + /// rule naming a reader the anonymous caller is not, and its pin is LEGACY (NULL + /// provenance) so the gate reads the page's own `pager.rules` rather than re-querying + /// per repo. Every fetch here is budget-shortened, so that repo arrives as the last + /// row of a shortened page: the boundary where a page carrying a rule set loaded for + /// a different row set would fail OPEN and serve a withheld object. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_scan_ceiling_below_page_size_bounds_the_query(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 4; + state.ipfs_max_legacy_scan_rows = 2; + // The probe, visit, and walk budgets stay at their defaults on purpose: the + // boundary repo below spends a probe, a visit, and a history walk that the + // denial-only rows do not, and a fixture that starved them would withhold it for + // a reason that has nothing to do with its rules. + + seed_root_denying_repos(&state, "capbelow", 6, 0).await; + // Both repos below are mirror rows, so `upsert_mirror_repo` stamps `now` and they + // sort after every 2020-stamped denial row. The boundary repo is seeded second, + // so `(created_at, id)` puts it eighth and last. + let (_, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6capbelow", + "holder", + b"past a ceiling below the page size\n", + ) + .await; + let holder_cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + let (_, withheld_oid) = seed_path_denying_repo( + &state, + tmp.path(), + "z6capbelow", + "boundary", + b"withheld at the boundary row\n", + ) + .await; + let withheld_cid = seed_legacy_pin_for_oid(&state, &withheld_oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.150:5000".parse().unwrap(); + + crate::api::ipfs::reset_scan_rows(); + crate::api::ipfs::reset_scan_limit(); + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&holder_cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + // Both counters sum over the whole request and are cleared only by their resets, + // so they are captured HERE, before the ladder below adds its own fetches. + let rows = crate::api::ipfs::scan_rows(); + let limit = crate::api::ipfs::scan_limit(); + assert_eq!( + limit, 2, + "one fetch, and it must ask the database for the ceiling (2), not the page \ + size (4). Asked for {limit}" + ); + assert_eq!( + rows, 2, + "a ceiling below the page size still bounds the selection exactly. Read \ + {rows} of 8 seeded rows" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the holder is row 7, past the ceiling, so this request must not serve it \ + and must tail to the retryable 503: {body}" + ); + assert_eq!( + body["error"], "search_incomplete", + "the shed must name the incomplete search: {body}" + ); + + // The ladder still reaches the holder: rungs covering rows 3-4, 5-6, then 7-8. + let bound = 8usize.div_ceil(2) + 1; + let mut token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung 1 must carry a continuation: {body}")), + ); + let mut served_at = None; + for step in 2..=bound { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&holder_cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")), + ); + } + assert!( + served_at.is_some(), + "a holder past the ceiling must still be served within ceil(8/2)+1 = {bound} \ + token-echoing requests; a ceiling that shortens the query must not shorten \ + the reach" + ); + + // The withheld blob gets its OWN full ladder, and is denied on every rung. + let mut token: Option = None; + for step in 1..=bound { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&withheld_cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + assert_ne!( + status, + StatusCode::OK, + "the /src/** rule names a reader the anonymous caller is not, so the \ + boundary repo's blob must never be served, including on the rung that \ + reaches it as the last row of a shortened page (step {step}): {body}" + ); + match continuation_of(&body) { + Some(t) => token = Some(t), + None => break, + } + } + } + + /// The positive control for the fixture above: the identical inventory with the + /// `/src/**` rule ABSENT serves the same blob through the same ladder. + /// + /// Its job is attribution. Without it, a not-served assertion is satisfied by any + /// fixture that never reaches the repo at all (a spent probe, a skipped walk, a + /// budget cut), so a RED there could not be read as "the rules decided". This test + /// is GREEN before and after the ceiling fix; only the pairing carries meaning. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_boundary_repo_serves_the_same_blob_without_the_path_rule( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 4; + state.ipfs_max_legacy_scan_rows = 2; + + seed_root_denying_repos(&state, "capbelowctl", 6, 0).await; + let (_, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6capbelowctl", + "holder", + b"past a ceiling below the page size\n", + ) + .await; + let _ = seed_legacy_pin_for_oid(&state, &holder_oid).await; + // Same recipe as the fixture above, minus the path-scoped rule. + let (_, allowed_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6capbelowctl", + "boundary", + b"withheld at the boundary row\n", + ) + .await; + let allowed_cid = seed_legacy_pin_for_oid(&state, &allowed_oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.151:5000".parse().unwrap(); + let bound = 8usize.div_ceil(2) + 1; + + let mut token: Option = None; + let mut served_at = None; + for step in 1..=bound { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&allowed_cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")), + ); + } + assert!( + served_at.is_some(), + "the boundary repo is public at \"/\" and the rule is what withholds its \ + blob, so with the rule absent the same blob must be served within {bound} \ + rungs" + ); + } + + /// Scenario 8 (#173 round 14, F4): a ceiling that is not a multiple of the page size + /// must shorten the LAST fetch to what is left of the budget. + /// + /// Page size 2, ceiling 3. The first fetch may ask for a full page; the second may + /// ask for one row only. A pager that asks for a page either way overshoots the + /// operator's ceiling by a page on every scan whose ceiling is not an exact multiple, + /// which is the general case. `scan_limit()` reads 2 + 1 = 3 for the fix and 2 + 2 = + /// 4 for a pager that trims after the query. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_scan_ceiling_not_a_page_multiple_shortens_the_last_fetch( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 3; + + seed_root_denying_repos(&state, "capodd", 6, 0).await; + // Seventh and last: `upsert_mirror_repo` stamps `now`, past every 2020 stamp. + let (_, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6capodd", + "holder", + b"past a ceiling that is not a page multiple\n", + ) + .await; + let holder_cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.152:5000".parse().unwrap(); + + crate::api::ipfs::reset_scan_rows(); + crate::api::ipfs::reset_scan_limit(); + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&holder_cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + // Captured before the ladder: both counters sum across the whole request. + let rows = crate::api::ipfs::scan_rows(); + let limit = crate::api::ipfs::scan_limit(); + assert_eq!( + limit, 3, + "two fetches, of 2 then 1: the second may ask only for the remaining budget. \ + Asked for {limit} rows in total" + ); + assert_eq!( + rows, 3, + "the ceiling (3) bounds the selection exactly, page size (2) or not. Read \ + {rows} of 7 seeded rows" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the holder is row 7, past the ceiling, so this request must not serve it \ + and must tail to the retryable 503: {body}" + ); + assert_eq!( + body["error"], "search_incomplete", + "the shed must name the incomplete search: {body}" + ); + + let bound = 7usize.div_ceil(3) + 1; + let mut token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung 1 must carry a continuation: {body}")), + ); + let mut served_at = None; + for step in 2..=bound { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&holder_cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")), + ); + } + assert!( + served_at.is_some(), + "a holder past the ceiling must still be served within ceil(7/3)+1 = {bound} \ + token-echoing requests" + ); + } + /// Seed `n` PUBLIC (root-READABLE) mirror rows in scan order, with disk paths that /// do not exist. /// From 21e687368336fd219d82308ee20b3d6bb018a3dc Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:39:05 -0500 Subject: [PATCH 58/77] fix(node): bound the legacy scan's query by the remaining row budget GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS accepts any value from one upward, but the pager always asked the database for a full page and the ceiling was only checked afterwards, in the scan loop. With the ceiling below the page size an anonymous request still selected and rule-loaded a whole page while holding walk admission, and any ceiling that is not a page multiple overshot by up to a page. Compute the remaining budget first and ask for min(page, remaining), so the page size stays a batching detail rather than an implicit floor under an operator ceiling. The exhaustion inference moves with it. A short page used to prove the table was exhausted; once the limit can be shortened by the budget it proves nothing, and marking the pager exhausted breaks at the top-of-loop arm that sits ahead of every ceiling arm, taints nothing, and mints no continuation. That path ends in a definitive 404 for content that exists, so the predicate now compares the returned rows against the limit actually sent. The budget computation sits above the timeout opener so the deadline-wrapping guard's lookback window is unaffected, and the limit is read from the AppState ceiling the row-ceiling arm reads, not the config field, so the arm and the query cannot disagree. Full suite 1059 green, fmt, clippy, and --locked clean. --- crates/gitlawb-node/src/api/ipfs.rs | 31 +++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 57cf8381..f2abb536 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -248,16 +248,30 @@ impl LegacyScanPager { .cursor .as_ref() .map(|(created_at, id)| (created_at.as_str(), id.as_str())); + // The row ceiling bounds what this REQUEST costs the database, so the ask is the + // smaller of one page and what is left of the budget. Capped in the LIMIT rather + // than by trimming the page once it has landed, because a trim bounds the result + // and leaves the selection, the transfer and the allocation already paid, which is + // the wrong half of the guarantee on an anonymously reachable route. + let remaining = state + .ipfs_max_legacy_scan_rows + .saturating_sub(self.fetched_rows); + let limit = state.ipfs_legacy_scan_page_rows.min(remaining); + // The caller's arm ordering is what guarantees this: `get_by_cid`'s row-ceiling + // arm breaks and mints a continuation before reaching this fetch once + // `fetched_rows >= ipfs_max_legacy_scan_rows`, so the budget always has room here. + debug_assert!( + limit >= 1, + "legacy scan LIMIT must ask for at least one row" + ); // Record the DB-facing ask before it is made. It sits above the timeout opener // because the INV-22 guard reads a fixed lookback from the query call for that // wrapper, and anything inserted inside the window eats its margin. #[cfg(test)] - note_scan_limit(state.ipfs_legacy_scan_page_rows); + note_scan_limit(limit); let page = match tokio::time::timeout( request_deadline.saturating_duration_since(std::time::Instant::now()), - state - .db - .list_repos_page_for_scan(after, state.ipfs_legacy_scan_page_rows as i64), + state.db.list_repos_page_for_scan(after, limit as i64), ) .await { @@ -277,8 +291,13 @@ impl LegacyScanPager { self.fetched_rows += page.len(); // Measured on the FULL page the query returned, before any rules cut shortens it: // this is the DB-facing row cost the row ceiling bounds, and a page that is short - // is a page with nothing behind it whatever the rules do. - if page.len() < state.ipfs_legacy_scan_page_rows { + // is a page with nothing behind it whatever the rules do. Compared against the + // limit ACTUALLY sent, not the page size: once the budget can shorten the ask, a + // page shorter than a full page proves nothing about the table, and marking the + // scan exhausted there breaks at the top-of-loop arm that sits ahead of every + // ceiling arm, taints nothing and mints no token, so existing content returns a + // false definitive 404. + if page.len() < limit { self.exhausted = true; } if page.is_empty() { From 1d37cb1ce91d552c645bcef725e541cfd477cf5c Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:13:29 -0500 Subject: [PATCH 59/77] test(gl): cover the scan-continuation resume ladder in gl ipfs get The node answers a truncated legacy CID scan with a 503 carrying a sealed continuation token, and gl ipfs get turns every non-success into a hard error, so content reachable only past a scan ceiling cannot be fetched with the first-party client at all. The old test asserted that terminal 503 as the desired behaviour, and its fixture body predates the token, so it is replaced rather than adjusted. Thirteen scenarios pin the contract: a resume that completes, the attempt cap, a wedged server that keeps minting fresh tokens for the same position, a missing or malformed token, a first-request overload, the signed resume, the wall-clock deadline, a mid-ladder rate limit that is terminal, a mid-ladder overload that is retried, the token length boundary pair, the token surfaced at every bounded terminal, and a caller-supplied continuation that starts from the token rather than the front. Hostile-input scenarios assert the surfaced message is capped and stripped of control and bidi characters and that a rejected token is never echoed back. Ladder fixtures answer Retry-After: 0 so the clamped waits are zero and the whole gl suite still runs in under a second in real time. The seam signatures land here so the tests compile and fail on their assertions: cmd_get delegates to cmd_get_inner, which threads a deadline, an attempt cap, and an optional starting continuation. Its body is unchanged and still bails on any non-success, so all fourteen new tests fail on the property each one names. --- crates/gl/src/ipfs_cmd.rs | 808 +++++++++++++++++++++++++++++++++++++- 1 file changed, 794 insertions(+), 14 deletions(-) diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index fa7a3f3f..d2c8e4c9 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -4,6 +4,7 @@ //! objects by their content-addressed CID. use std::path::PathBuf; +use std::time::Duration; use anyhow::{Context, Result}; use clap::{Args, Subcommand}; @@ -89,7 +90,41 @@ async fn cmd_list(node: String, dir: Option) -> Result<()> { Ok(()) } +/// Automatic resumes attempted after the initial request when the node reports a +/// truncated legacy scan, so at most `MAX_SCAN_RESUMES + 1` node calls per invocation. +const MAX_SCAN_RESUMES: usize = 8; + +/// Wall-clock budget for a whole `gl ipfs get`, resumes included. +const SCAN_DEADLINE: Duration = Duration::from_secs(60); + +/// Mirror a stderr diagnostic into a per-thread buffer under `cfg(test)` so the +/// command-level tests can assert on what the caller is actually told. Callers +/// see the same line either way; only the test-visible copy is conditional. +fn diag(msg: &str) { + eprintln!("{msg}"); + #[cfg(test)] + tests::record_diag(msg); +} + async fn cmd_get(cid: String, node: String, dir: Option) -> Result<()> { + cmd_get_inner(cid, node, dir, None, SCAN_DEADLINE, MAX_SCAN_RESUMES).await +} + +/// The body of `gl ipfs get`, with the bounds and the starting continuation as +/// parameters so tests can drive the resume ladder without waiting out the shipped +/// defaults. `cmd_get` supplies those defaults. +/// +/// The resume loop itself is not wired yet: `continuation`, `deadline`, and `cap` +/// are accepted here so the ladder tests can address the seam, and the request +/// behaviour below is still the single-shot, bail-on-any-non-success one. +async fn cmd_get_inner( + cid: String, + node: String, + dir: Option, + _continuation: Option, + _deadline: Duration, + _cap: usize, +) -> Result<()> { // #173 (F5): the resolver now serves path-scoped objects to authorized readers, // so sign with an available identity like `gl ipfs list` — otherwise an owner or // listed reader gets the opaque anonymous 404 for content they can read. @@ -127,10 +162,16 @@ async fn cmd_get(cid: String, node: String, dir: Option) -> Result<()> // Print headers for diagnostics let headers = resp.headers().clone(); if let Some(git_hash) = headers.get("x-git-hash") { - eprintln!("x-git-hash: {}", git_hash.to_str().unwrap_or("?")); + diag(&format!( + "x-git-hash: {}", + git_hash.to_str().unwrap_or("?") + )); } if let Some(content_cid) = headers.get("x-content-cid") { - eprintln!("x-content-cid: {}", content_cid.to_str().unwrap_or("?")); + diag(&format!( + "x-content-cid: {}", + content_cid.to_str().unwrap_or("?") + )); } // Write raw bytes to stdout (allows piping to files or other tools) @@ -156,6 +197,119 @@ fn encode_cid_segment(cid: &str) -> String { #[cfg(test)] mod tests { use super::*; + use std::cell::RefCell; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + use std::time::Instant; + + thread_local! { + /// Test-visible copy of the stderr diagnostics `diag` emits. `#[tokio::test]` + /// runs the future on the test's own thread, so a thread-local is enough. + static DIAG: RefCell = const { RefCell::new(String::new()) }; + } + + pub(super) fn record_diag(msg: &str) { + DIAG.with(|d| { + let mut d = d.borrow_mut(); + d.push_str(msg); + d.push('\n'); + }); + } + + fn reset_diag() { + DIAG.with(|d| d.borrow_mut().clear()); + } + + fn diag_text() -> String { + DIAG.with(|d| d.borrow().clone()) + } + + /// Everything the caller is told about a failed get: the stderr diagnostics plus + /// the error itself (main renders both). `{:#}` flattens the anyhow context chain + /// onto one line, so a message carried in a context layer is still covered. + fn told(err: &anyhow::Error) -> String { + format!("{}{err:#}", diag_text()) + } + + /// Width of a real continuation today (see the node's scan_token module): 668 + /// base64url-no-pad characters. The tests build tokens of that width so the + /// fixtures look like the wire, not like a placeholder. + const TOKEN_LEN: usize = 668; + + fn token_of_len(seed: &str, len: usize) -> String { + let mut t = String::from(seed); + while t.len() < len { + t.push('A'); + } + t.truncate(len); + t + } + + fn make_token(seed: &str) -> String { + token_of_len(seed, TOKEN_LEN) + } + + /// The `scan` query value of an incoming request, if it carries one. + fn scan_of(path_and_query: &str) -> Option { + let (_, query) = path_and_query.split_once('?')?; + query + .split('&') + .find_map(|kv| kv.strip_prefix("scan=")) + .map(str::to_string) + } + + /// Derive the NEXT continuation from the one the client just echoed, so every + /// response on a ladder carries a different token (a real node re-seals with a + /// fresh nonce every time, and a fixed replayed body would hide that). + fn next_token(echoed: Option<&str>) -> String { + let n = echoed + .map(|t| { + t.trim_end_matches('A') + .trim_start_matches('t') + .parse::() + .unwrap_or(0) + }) + .unwrap_or(0); + make_token(&format!("t{}", n + 1)) + } + + /// A node message crafted to reach the terminal: a raw DEL (a control character + /// JSON permits unescaped), a JSON-escaped ESC/CSI sequence, a raw bidi override, + /// and a long tail so a missing length cap shows up. + fn hostile_msg() -> String { + format!("boom \u{7f} \\u001b[31m \u{202e} {}", "x".repeat(5000)) + } + + fn incomplete_body(continuation: Option<&str>, msg: &str) -> String { + match continuation { + Some(t) => { + format!(r#"{{"error":"search_incomplete","message":"{msg}","continuation":"{t}"}}"#) + } + None => format!(r#"{{"error":"search_incomplete","message":"{msg}"}}"#), + } + } + + fn has_control_or_bidi(s: &str) -> bool { + s.chars() + .any(|c| c.is_control() || gitlawb_core::sanitize::is_bidi_format(c)) + } + + /// A bare re-run restarts the scan at row 0 and re-spends the caller's per-IP + /// budget, so any "run it again" phrasing is only honest when the token that + /// makes progress is right there with it. + fn implies_bare_rerun(text: &str, token: &str) -> bool { + let lower = text.to_lowercase(); + [ + "try again", + "re-run", + "rerun", + "run it again", + "retry the command", + ] + .iter() + .any(|p| lower.contains(p)) + && !text.contains(token) + } /// Seed a keypair into a temp dir the way `load_keypair_from_dir` expects, /// then return the dir handle (keeps it alive for the test's duration). @@ -329,34 +483,660 @@ mod tests { m.assert_async().await; } - /// #173 (INV-8) must-not: the node's new 503 "search incomplete" (the legacy CID - /// scan hit its bound and could not prove absence) must surface as an actionable - /// Err naming the status, NOT be rendered as an empty/"not found" success — a - /// retryable outcome the caller has to see. Mirrors the 404 denial case for the - /// bounded-search response the resolver now emits. + // #173 (F3): a truncated legacy scan comes back as 503 `search_incomplete` with a + // sealed continuation token. The command must follow that token instead of + // dead-ending, under an attempt cap, a wall-clock deadline, and a clamped + // Retry-After, and every terminal that still holds a token must hand it back with + // the invocation that resumes from it. The ladder fixtures answer with + // `Retry-After: 0` so the clamped sleeps are zero and a nine-call ladder stays + // sub-second in real time. + + /// Scenario 1. A `search_incomplete` 503 carrying a valid continuation is resumed: + /// the second request repeats the CID with `?scan=` (percent-encoding is the + /// identity over the base64url alphabet, so the echo is byte-identical) and the + /// content it returns is written. #[tokio::test] - async fn test_cmd_get_search_incomplete_503_is_error() { + async fn test_cmd_get_resumes_search_incomplete_with_continuation() { + reset_diag(); let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + + let m1 = server + .mock("GET", "/ipfs/bafkreiresume") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", "/ipfs/bafkreiresume") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_body("object bytes") + .expect(1) + .create_async() + .await; + + cmd_get("bafkreiresume".to_string(), server.url(), None) + .await + .expect("a search_incomplete 503 carrying a continuation must resume, not bail"); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 2. A node that keeps truncating stops at the attempt cap: 8 automatic + /// resumes after the initial request, 9 node calls in all. The give-up names the + /// incomplete result and the cap, and hands back the token still held with the + /// invocation that resumes from it. + #[tokio::test] + async fn test_cmd_get_resume_ladder_stops_at_attempt_cap() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let calls = Arc::new(AtomicUsize::new(0)); + let c = calls.clone(); let m = server - .mock("GET", "/ipfs/bafkreiincomplete") + .mock("GET", mockito::Matcher::Any) .with_status(503) .with_header("content-type", "application/json") - .with_body(r#"{"error":"search_incomplete","message":"CID search incomplete — retry"}"#) + .with_header("retry-after", "0") + .with_body_from_request(move |req| { + c.fetch_add(1, Ordering::SeqCst); + let next = next_token(scan_of(req.path_and_query()).as_deref()); + incomplete_body(Some(&next), "scan truncated").into_bytes() + }) + .expect(9) .create_async() .await; - let err = cmd_get("bafkreiincomplete".to_string(), server.url(), None) + let err = cmd_get("bafkreicap".to_string(), server.url(), None) .await - .expect_err("a 503 incomplete-search must be an error, not masked as not-found"); + .expect_err("a ladder that never completes must end in an error"); + let told = told(&err); + let held = make_token("t9"); + + assert_eq!( + calls.load(Ordering::SeqCst), + 9, + "cap is 8 resumes after the initial request, so exactly 9 node calls" + ); assert!( - err.to_string().contains("503"), - "error should mention the status, got: {err}" + told.to_lowercase().contains("incomplete"), + "the give-up must name the incomplete result, got: {told}" + ); + assert!( + told.contains('8'), + "the give-up must name the resume cap, got: {told}" + ); + assert!( + told.contains(&held), + "the still-held continuation must be surfaced, got: {told}" + ); + assert!( + told.contains(&format!("--scan {held}")), + "the exact resuming invocation must be surfaced, got: {told}" + ); + assert!( + !implies_bare_rerun(&told, &held), + "a bare re-run restarts at row 0, so the wording must not imply it helps: {told}" + ); + + m.assert_async().await; + } + + /// Scenario 3. A wedged node (every response a fresh token for the same position) + /// is indistinguishable from slow progress at the client, because tokens are + /// nonce-randomized ciphertext. The cap is what ends it, and that is the whole + /// assertion: the ladder stops at 9 calls with the explicit incomplete report. + #[tokio::test] + async fn test_cmd_get_wedged_ladder_still_stops_at_attempt_cap() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let calls = Arc::new(AtomicUsize::new(0)); + let c = calls.clone(); + + let m = server + .mock("GET", mockito::Matcher::Any) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body_from_request(move |_req| { + let n = c.fetch_add(1, Ordering::SeqCst) + 1; + // A distinct token every time, none of them advancing the cursor. + incomplete_body(Some(&make_token(&format!("w{n}"))), "scan truncated").into_bytes() + }) + .expect(9) + .create_async() + .await; + + let err = cmd_get("bafkreiwedged".to_string(), server.url(), None) + .await + .expect_err("a wedged ladder must end in an error"); + let told = told(&err); + + assert_eq!( + calls.load(Ordering::SeqCst), + 9, + "the cap is the only bound on a wedged ladder, so exactly 9 node calls" + ); + assert!( + told.to_lowercase().contains("incomplete"), + "the give-up must name the incomplete result, got: {told}" + ); + + m.assert_async().await; + } + + /// Scenario 4. A `search_incomplete` 503 with no continuation is terminal: there is + /// nothing to resume from, and the message says so rather than reporting a bare + /// status. Exactly one node call. + #[tokio::test] + async fn test_cmd_get_search_incomplete_without_continuation_is_terminal() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + + let m1 = server + .mock("GET", "/ipfs/bafkreinotoken") + .with_status(503) + .with_header("content-type", "application/json") + .with_body(incomplete_body(None, "scan truncated")) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", mockito::Matcher::Regex("scan=".to_string())) + .expect(0) + .create_async() + .await; + + let err = cmd_get("bafkreinotoken".to_string(), server.url(), None) + .await + .expect_err("a truncation with no continuation must be an error"); + let told = told(&err); + + assert!( + told.to_lowercase().contains("continuation"), + "the terminal must name the missing continuation, got: {told}" + ); + assert!( + told.contains("503"), + "the terminal must still name the status, got: {told}" + ); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 5. An overload 503 on the FIRST request holds no token, so there is + /// nothing to resume: terminal, one call, and the node's text reaches the terminal + /// sanitized and length-capped rather than verbatim. + #[tokio::test] + async fn test_cmd_get_first_request_overload_503_is_terminal_and_sanitized() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + + let m1 = server + .mock("GET", "/ipfs/bafkreioverload") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(format!( + r#"{{"error":"overloaded","message":"{}"}}"#, + hostile_msg() + )) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", mockito::Matcher::Regex("scan=".to_string())) + .expect(0) + .create_async() + .await; + + let err = cmd_get("bafkreioverload".to_string(), server.url(), None) + .await + .expect_err("a first-request overload must be an error"); + let told = told(&err); + + assert!( + !has_control_or_bidi(&told), + "node text must be sanitized before it reaches the terminal, got: {told:?}" + ); + assert!( + told.chars().count() < 600, + "node text must be length-capped, got {} chars", + told.chars().count() + ); + assert!( + told.contains("503"), + "the terminal must name the status, got: {told}" + ); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 6. The resumed request is signed like the first one, and the signature + /// covers the query: the token joins the path binding before signing, so the mock + /// matching both the `scan=` query and the RFC 9421 headers is the one served. + #[tokio::test] + async fn test_cmd_get_resumed_request_is_signed() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let keystore = seed_keystore(); + let t = make_token("t1"); + + let m1 = server + .mock("GET", "/ipfs/bafkreisigned") + .match_header("signature", mockito::Matcher::Any) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", "/ipfs/bafkreisigned") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .match_header("signature", mockito::Matcher::Any) + .match_header("signature-input", mockito::Matcher::Any) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_body("object bytes") + .expect(1) + .create_async() + .await; + + cmd_get( + "bafkreisigned".to_string(), + server.url(), + Some(keystore.path().to_path_buf()), + ) + .await + .expect("the resumed request must be signed and served"); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 7. An oversized, hostile `search_incomplete` body still terminates + /// cleanly: the surfaced message is capped and free of control and bidi characters. + #[tokio::test] + async fn test_cmd_get_hostile_incomplete_body_is_capped_and_sanitized() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + + let m = server + .mock("GET", "/ipfs/bafkreihostilebody") + .with_status(503) + .with_header("content-type", "application/json") + .with_body(incomplete_body(None, &hostile_msg())) + .expect(1) + .create_async() + .await; + + let err = cmd_get("bafkreihostilebody".to_string(), server.url(), None) + .await + .expect_err("a hostile truncation body must still be an error"); + let told = told(&err); + + assert!( + !has_control_or_bidi(&told), + "node text must be sanitized before it reaches the terminal, got: {told:?}" + ); + assert!( + told.chars().count() < 600, + "node text must be length-capped, got {} chars", + told.chars().count() ); m.assert_async().await; } + /// Scenario 8. A continuation the node chose but that fails validation (`#`, a + /// newline, `&`) never enters the signed path: terminal exactly like a missing + /// token, no second request, and the rejected token is never echoed into the + /// message (the bound is named instead). + #[tokio::test] + async fn test_cmd_get_hostile_continuation_token_is_rejected() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + + let m1 = server + .mock("GET", "/ipfs/bafkreihostiletoken") + .with_status(503) + .with_header("content-type", "application/json") + .with_body(format!( + r#"{{"error":"search_incomplete","message":"{}","continuation":"abc#\ndef&ghi"}}"#, + hostile_msg() + )) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", mockito::Matcher::Regex("scan=".to_string())) + .expect(0) + .create_async() + .await; + + let err = cmd_get("bafkreihostiletoken".to_string(), server.url(), None) + .await + .expect_err("a malformed continuation must be terminal"); + let told = told(&err); + + assert!( + !told.contains("abc#") && !told.contains("def&ghi"), + "a rejected token must never be echoed into the message, got: {told}" + ); + assert!( + !has_control_or_bidi(&told), + "node text must be sanitized before it reaches the terminal, got: {told:?}" + ); + assert!( + told.chars().count() < 600, + "node text must be length-capped, got {} chars", + told.chars().count() + ); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 9. A mid-ladder 429 is terminal: the fanout limiter's window is an + /// hour, so its Retry-After cannot be honored inside one invocation. The message + /// names rate limiting (not truncation), is sanitized and capped, and the token + /// still held comes back with the invocation that resumes from it. + #[tokio::test] + async fn test_cmd_get_mid_ladder_429_is_terminal_and_surfaces_token() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + + let m1 = server + .mock("GET", "/ipfs/bafkreithrottled") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", "/ipfs/bafkreithrottled") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(429) + .with_header("content-type", "application/json") + .with_header("retry-after", "3600") + .with_body(format!( + r#"{{"error":"rate_limited","message":"{}"}}"#, + hostile_msg() + )) + .expect(1) + .create_async() + .await; + + let err = cmd_get("bafkreithrottled".to_string(), server.url(), None) + .await + .expect_err("a mid-ladder 429 must be an error"); + let told = told(&err); + + assert!( + told.to_lowercase().contains("rate limit"), + "the terminal must name rate limiting, distinct from the truncation wording, got: {told}" + ); + assert!( + !has_control_or_bidi(&told), + "node text must be sanitized before it reaches the terminal, got: {told:?}" + ); + assert!( + told.chars().count() < 600, + "node text must be length-capped, got {} chars", + told.chars().count() + ); + assert!( + told.contains(&t) && told.contains(&format!("--scan {t}")), + "the still-held continuation and its resuming invocation must be surfaced, got: {told}" + ); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 10. A mid-ladder overload 503 (no `search_incomplete` code, token still + /// held) is retried, not terminal: its three sources are transient, the node itself + /// says to retry shortly, and nothing accumulates per IP on that path. The ladder + /// continues on the same token and completes in three calls. + #[tokio::test] + async fn test_cmd_get_mid_ladder_overload_503_is_retried() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + + let m1 = server + .mock("GET", "/ipfs/bafkreimidoverload") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + // Both of the next two match `scan=T`; mockito serves the first one that still + // has hits outstanding, so registration order sequences the overload then the + // success. + let m2 = server + .mock("GET", "/ipfs/bafkreimidoverload") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(r#"{"error":"overloaded","message":"busy, retry shortly"}"#) + .expect(1) + .create_async() + .await; + let m3 = server + .mock("GET", "/ipfs/bafkreimidoverload") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_body("object bytes") + .expect(1) + .create_async() + .await; + + cmd_get("bafkreimidoverload".to_string(), server.url(), None) + .await + .expect("a mid-ladder overload must be retried on the held token, not terminal"); + + m1.assert_async().await; + m2.assert_async().await; + m3.assert_async().await; + } + + /// Scenario 11. The wall-clock deadline bounds the whole loop, and it bounds each + /// request's own timeout, so the composed worst case is the deadline plus one + /// clamped wait. Injected through the seam because the shipped 60s is unreachable + /// under the 5s clamp and 8 resumes. + #[tokio::test] + async fn test_cmd_get_resume_ladder_stops_at_wall_clock_deadline() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let calls = Arc::new(AtomicUsize::new(0)); + let last = Arc::new(Mutex::new(String::new())); + let c = calls.clone(); + let l = last.clone(); + + let m = server + .mock("GET", mockito::Matcher::Any) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "1") + .with_body_from_request(move |req| { + c.fetch_add(1, Ordering::SeqCst); + let next = next_token(scan_of(req.path_and_query()).as_deref()); + *l.lock().unwrap() = next.clone(); + incomplete_body(Some(&next), "scan truncated").into_bytes() + }) + .expect_at_least(2) + .create_async() + .await; + + let started = Instant::now(); + let err = cmd_get_inner( + "bafkreideadline".to_string(), + server.url(), + None, + None, + Duration::from_millis(2500), + MAX_SCAN_RESUMES, + ) + .await + .expect_err("a ladder that outruns the deadline must end in an error"); + let elapsed = started.elapsed(); + let told = told(&err); + let held = last.lock().unwrap().clone(); + + assert!( + told.to_lowercase().contains("deadline"), + "the give-up must name the deadline, not the cap, got: {told}" + ); + let calls = calls.load(Ordering::SeqCst); + assert!( + (2..9).contains(&calls), + "the deadline must stop the ladder before the cap, made {calls} calls" + ); + assert!( + elapsed < Duration::from_secs(9), + "composed worst case is the 2.5s deadline plus one 5s clamped wait, took {elapsed:?}" + ); + assert!( + told.contains(&held) && told.contains(&format!("--scan {held}")), + "the still-held continuation and its resuming invocation must be surfaced, got: {told}" + ); + + m.assert_async().await; + } + + /// Scenario 12, lower half of the boundary pair: a 2048-character token is inside + /// the accepted bound and is resumed with. + #[tokio::test] + async fn test_cmd_get_accepts_2048_char_continuation() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = token_of_len("t1", 2048); + + let m1 = server + .mock("GET", "/ipfs/bafkreibound") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", "/ipfs/bafkreibound") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_body("object bytes") + .expect(1) + .create_async() + .await; + + cmd_get("bafkreibound".to_string(), server.url(), None) + .await + .expect("a 2048-character token is within the bound and must be resumed with"); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 12, upper half: one character past the bound is rejected, terminal + /// exactly like a missing token, and never echoed back. + #[tokio::test] + async fn test_cmd_get_rejects_2049_char_continuation() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = token_of_len("t1", 2049); + + let m1 = server + .mock("GET", "/ipfs/bafkreioverbound") + .with_status(503) + .with_header("content-type", "application/json") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", mockito::Matcher::Regex("scan=".to_string())) + .expect(0) + .create_async() + .await; + + let err = cmd_get("bafkreioverbound".to_string(), server.url(), None) + .await + .expect_err("an over-bound token must be terminal"); + let told = told(&err); + + assert!( + !told.contains(&t), + "a rejected token must never be echoed into the message, got: {told}" + ); + assert!( + told.chars().count() < 600, + "node text must be length-capped, got {} chars", + told.chars().count() + ); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 13. A caller-supplied continuation is a resume INPUT: the very first + /// request carries `?scan=`, so an invocation picked up from a previous + /// terminal starts where that one stopped instead of walking from row 0 again. + #[tokio::test] + async fn test_cmd_get_caller_supplied_continuation_starts_from_token() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t7"); + + let front = server + .mock("GET", "/ipfs/bafkreisupplied") + .expect(0) + .create_async() + .await; + let resumed = server + .mock("GET", "/ipfs/bafkreisupplied") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_body("object bytes") + .expect(1) + .create_async() + .await; + + let res = cmd_get_inner( + "bafkreisupplied".to_string(), + server.url(), + None, + Some(t.clone()), + SCAN_DEADLINE, + MAX_SCAN_RESUMES, + ) + .await; + + front.assert_async().await; + resumed.assert_async().await; + res.expect("a supplied continuation must be used, not ignored"); + } + /// #173 review (F1): a base64 CID (multibase prefix 'm') can contain '/', '+', /// and '='. The client must percent-encode it into ONE path segment before /// building and signing `/ipfs/`; otherwise the '/' splits the target so From 9a8d190492315cf767807dd1a98470debff4bd25 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:47:41 -0500 Subject: [PATCH 60/77] fix(gl): follow scan continuations in gl ipfs get, bounded three ways A truncated legacy CID scan answers 503 with a sealed continuation token, and the command turned that into a hard error, so content past a scan ceiling was unreachable from the first-party client. It now follows the continuation automatically. Three bounds, each protecting a different resource. The attempt cap bounds the work asked of the node. A wall-clock deadline bounds the caller's own time and is passed into each request as that request's timeout, so a slow-but-alive node cannot outlast it. The per-wait clamp bounds a single Retry-After, so a hostile value cannot stall the client. There is deliberately no non-advancing-token check: tokens are nonce-randomized ciphertext, so two seals of one position are never byte-equal and a client cannot detect a wedged ladder by comparing them. Classification happens at one site with no wildcard to retry. Status is checked before the body is read. A rate limit is terminal on status alone, since its window is an hour and its Retry-After cannot be honored inside one invocation. An overload mid-ladder is retried on the still-held token, because that shed releases its permit when the request ends and the node advertises a retry. Any other status or code is terminal through the default arm, including an overload on the first request, where no token is held and there is nothing to resume. Every terminal message is capped and sanitized before it reaches the terminal, and a rejected token is never echoed back; the bound is named instead. Tokens are accepted only as non-empty base64url within a generous ceiling, so a future change to the sealed layout cannot silently disable resume. Where a bound stops the ladder, the still-held token is printed with the invocation that resumes from it, rather than wording that implies a bare re-run makes progress. gl suite 332 green, fmt and clippy clean. --- crates/gl/src/ipfs_cmd.rs | 212 ++++++++++++++++++++++++++++++++++---- 1 file changed, 190 insertions(+), 22 deletions(-) diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index d2c8e4c9..6d2b2b72 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -10,7 +10,7 @@ use anyhow::{Context, Result}; use clap::{Args, Subcommand}; use serde_json::Value; -use crate::http::NodeClient; +use crate::http::{read_body_capped, sanitize_node_msg, NodeClient}; #[derive(Args)] pub struct IpfsArgs { @@ -97,6 +97,20 @@ const MAX_SCAN_RESUMES: usize = 8; /// Wall-clock budget for a whole `gl ipfs get`, resumes included. const SCAN_DEADLINE: Duration = Duration::from_secs(60); +/// Longest single wait honored between attempts, whatever `Retry-After` asks for. +/// The node picks that number, so an unclamped sleep would let a hostile one stall +/// the client for as long as it likes. +const MAX_RETRY_AFTER: Duration = Duration::from_secs(5); + +/// Wait used when a retryable response carries no usable `Retry-After`. +const DEFAULT_RETRY_AFTER: Duration = Duration::from_secs(1); + +/// Generous ceiling on a continuation token. Real tokens are fixed-width (668 +/// base64url characters today), but the sealed layout has already changed once and +/// a rejected token is terminal, so a tight bound would silently kill resume on a +/// future version bump. +const MAX_CONTINUATION_LEN: usize = 2048; + /// Mirror a stderr diagnostic into a per-thread buffer under `cfg(test)` so the /// command-level tests can assert on what the caller is actually told. Callers /// see the same line either way; only the test-visible copy is conditional. @@ -113,17 +127,13 @@ async fn cmd_get(cid: String, node: String, dir: Option) -> Result<()> /// The body of `gl ipfs get`, with the bounds and the starting continuation as /// parameters so tests can drive the resume ladder without waiting out the shipped /// defaults. `cmd_get` supplies those defaults. -/// -/// The resume loop itself is not wired yet: `continuation`, `deadline`, and `cap` -/// are accepted here so the ladder tests can address the seam, and the request -/// behaviour below is still the single-shot, bail-on-any-non-success one. async fn cmd_get_inner( cid: String, node: String, dir: Option, - _continuation: Option, - _deadline: Duration, - _cap: usize, + continuation: Option, + deadline: Duration, + cap: usize, ) -> Result<()> { // #173 (F5): the resolver now serves path-scoped objects to authorized readers, // so sign with an available identity like `gl ipfs list` — otherwise an owner or @@ -147,19 +157,122 @@ async fn cmd_get_inner( // route nor points at the intended target. Percent-encode the CID as exactly // one path segment so the signed and sent target agree and the server's // `Path` extractor decodes it back to the original CID. - let path = format!("/ipfs/{}", encode_cid_segment(&cid)); - let resp = client - .get_authed(&path) - .await - .with_context(|| format!("failed to fetch CID {cid} from {node}"))?; + let encoded_cid = encode_cid_segment(&cid); + + // A caller-supplied continuation reaches the same signed target as a node-chosen + // one, so it clears the same bar before the first request. + let mut token = match continuation { + Some(t) if valid_continuation(&t) => Some(t), + Some(_) => anyhow::bail!( + "the supplied continuation is not a resume token: \ + expected 1 to {MAX_CONTINUATION_LEN} base64url characters" + ), + None => None, + }; - let status = resp.status(); - if !status.is_success() { - let body = resp.text().await.unwrap_or_default(); - anyhow::bail!("node returned {status}: {body}"); + // One deadline for the whole ladder, captured before the first request and used + // both as the loop's bound and as each request's own timeout, so the composed + // worst case is the deadline plus one clamped wait rather than the deadline plus + // the client's blanket 30s. + let start = tokio::time::Instant::now(); + let mut requests = 0usize; + loop { + if requests > cap { + diag(&format!( + "warning: the node's legacy scan is still incomplete after {cap} automatic \ + resumes; the object may sit beyond the rows scanned so far" + )); + surface_resume(&cid, token.as_deref()); + anyhow::bail!( + "gave up on an incomplete scan for CID {cid} after {requests} node calls" + ); + } + let remaining = deadline.saturating_sub(start.elapsed()); + if remaining.is_zero() { + return Err(deadline_reached(&cid, token.as_deref(), deadline)); + } + + // The token joins the single `path` binding BEFORE `get_authed` signs, so the + // signature covers the query string and the bytes signed are the bytes sent. + // Percent-encoding is the identity over the accepted alphabet; it is here for + // the value that is not. + let path = match &token { + Some(t) => format!("/ipfs/{encoded_cid}?scan={}", urlencoding::encode(t)), + None => format!("/ipfs/{encoded_cid}"), + }; + let resp = match tokio::time::timeout(remaining, client.get_authed(&path)).await { + Ok(r) => r.with_context(|| format!("failed to fetch CID {cid} from {node}"))?, + Err(_) => return Err(deadline_reached(&cid, token.as_deref(), deadline)), + }; + requests += 1; + + // Status first, before any body read. + let status = resp.status(); + if status.is_success() { + return write_object(resp).await; + } + if status == reqwest::StatusCode::TOO_MANY_REQUESTS { + // Terminal on the status alone: the fanout limiter's window is an hour, so + // the wait it advertises cannot be honored inside one invocation, and + // retrying only deepens the shedding the ladder itself caused. + diag( + "warning: the node is rate limiting this scan, so the result is incomplete; \ + its limit window outlasts a single invocation", + ); + surface_resume(&cid, token.as_deref()); + anyhow::bail!("node returned {status}: rate limited"); + } + + let retry_after = parse_retry_after(resp.headers()); + let raw = read_body_capped(resp, 8 * 1024).await; + let parsed = serde_json::from_str::(&raw).ok(); + let code = parsed.as_ref().and_then(|v| v["error"].as_str()); + let node_msg = parsed + .as_ref() + .and_then(|v| v["message"].as_str()) + .unwrap_or(raw.as_str()); + let offered = parsed.as_ref().and_then(|v| v["continuation"].as_str()); + + // The one classification site. The node's error code picks the arm and the + // default is terminal, so an unrecognized code can never resolve to a retry. + let resume_with = match code { + Some("search_incomplete") => offered + .filter(|t| valid_continuation(t)) + .map(str::to_string), + // A mid-ladder overload sheds a request whose permit is released at request + // end and asks the caller back shortly, so the ladder continues on the token + // it already holds. With no token there is nothing to resume, which falls to + // the default arm below. + Some(_) | None if status == reqwest::StatusCode::SERVICE_UNAVAILABLE => token.clone(), + _ => None, + }; + + let Some(next) = resume_with else { + let msg = sanitize_node_msg(node_msg); + if code == Some("search_incomplete") { + // Naming the bound, never echoing the value: a rejected token is + // node-chosen text and has no business in a terminal message. + let why = if offered.is_some() { + format!( + "the continuation it offered is not a resume token \ + (expected 1 to {MAX_CONTINUATION_LEN} base64url characters)" + ) + } else { + "it offered no continuation token".to_string() + }; + anyhow::bail!("node returned {status} with the scan incomplete and {why}: {msg}"); + } + anyhow::bail!("node returned {status}: {msg}"); + }; + + tokio::time::sleep(retry_after.min(MAX_RETRY_AFTER)).await; + token = Some(next); } +} - // Print headers for diagnostics +/// Write a successful response: diagnostics to stderr, raw bytes to stdout so the +/// output stays pipeable. +async fn write_object(resp: reqwest::Response) -> Result<()> { let headers = resp.headers().clone(); if let Some(git_hash) = headers.get("x-git-hash") { diag(&format!( @@ -174,7 +287,6 @@ async fn cmd_get_inner( )); } - // Write raw bytes to stdout (allows piping to files or other tools) let bytes = resp.bytes().await.context("failed to read response body")?; use std::io::Write; std::io::stdout() @@ -184,6 +296,52 @@ async fn cmd_get_inner( Ok(()) } +/// Report the wall-clock give-up and hand the caller their token back. +fn deadline_reached(cid: &str, token: Option<&str>, deadline: Duration) -> anyhow::Error { + diag(&format!( + "warning: the node's legacy scan is still incomplete at the {}s deadline; \ + the object may sit beyond the rows scanned so far", + deadline.as_secs_f32() + )); + surface_resume(cid, token); + anyhow::anyhow!("gave up on an incomplete scan for CID {cid} at the wall-clock deadline") +} + +/// Hand back the token that still points at where the scan stopped, together with +/// the invocation that resumes from it. A bare re-run restarts at row 0, reproduces +/// the same truncation, and re-spends the caller's per-IP budget, so the token is +/// the only thing that makes progress. +fn surface_resume(cid: &str, token: Option<&str>) { + if let Some(t) = token { + diag(&format!( + "resume from where this stopped: gl ipfs get {cid} --scan {t}" + )); + } +} + +/// `Retry-After` in delta-seconds. Absent, non-numeric, or an HTTP-date all fall +/// back to one second; the caller clamps whatever comes back. +fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> Duration { + headers + .get(reqwest::header::RETRY_AFTER) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.trim().parse::().ok()) + .map(Duration::from_secs) + .unwrap_or(DEFAULT_RETRY_AFTER) +} + +/// A continuation is node-chosen and goes straight into a signed request target, so +/// accept only the alphabet the node's sealer emits. A value carrying `#`, `&`, `?`, +/// `/`, whitespace, or control bytes could make the URL reqwest parses differ from +/// the bytes signed. +fn valid_continuation(token: &str) -> bool { + !token.is_empty() + && token.len() <= MAX_CONTINUATION_LEN + && token + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') +} + /// Percent-encode a CID so it occupies exactly one path segment of `/ipfs/`. /// `urlencoding::encode` escapes every byte outside the RFC 3986 unreserved set /// (ALPHA / DIGIT / `-._~`), so the base64-CID characters that would otherwise @@ -212,7 +370,12 @@ mod tests { DIAG.with(|d| { let mut d = d.borrow_mut(); d.push_str(msg); - d.push('\n'); + // A space, not a newline: the `has_control_or_bidi` assertions run over the + // whole telling, so a newline the harness inserts itself would make them + // fire on ANY stderr diagnostic and turn "node text is sanitized" into + // "nothing was printed to stderr", which R21 requires. A node-supplied + // control character is still caught. + d.push(' '); }); } @@ -893,10 +1056,15 @@ mod tests { !has_control_or_bidi(&told), "node text must be sanitized before it reaches the terminal, got: {told:?}" ); + // The length bound is scoped to the error text, not to `told`: R21 requires a + // stderr line carrying the still-held 668-character token, so no implementation + // can keep the whole telling under 600 characters. The error text is where an + // uncapped node body would land on this path, so the property still binds. + let reported = format!("{err:#}"); assert!( - told.chars().count() < 600, + reported.chars().count() < 600, "node text must be length-capped, got {} chars", - told.chars().count() + reported.chars().count() ); assert!( told.contains(&t) && told.contains(&format!("--scan {t}")), From 181f6178a0e0f5b9506093fa7579ad6ef8dd2ff5 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:02:34 -0500 Subject: [PATCH 61/77] fix(gl): accept the resume token the incomplete-scan message tells you to pass The bounded terminals printed "gl ipfs get --scan ", but the command took no --scan argument, so following that advice failed on an unexpected argument and the resume input existed only behind a private seam. The flag now exists and forwards to the same validation a node-supplied token gets, which is what makes surfacing the token worth anything: without it a stopped ladder restarts from the front on every rerun. Three test fixes ride along, all found by running the mutation matrix rather than by reading. The deadline scenario asserted its message "must name the deadline" while its fixture CID was bafkreideadline, so any message echoing the CID satisfied it. Under a mutant that removed the deadline the ladder ran to the cap and that assertion still passed. The CID is renamed so the assertion can only pass when the message genuinely names the deadline, and the mutant now reddens on it instead of on the call count. The default classification arm was certified only for the token-less case: no fixture reached it with a token already held, so degrading it to retry was an equivalent mutation against the whole suite. A scenario now drives an unrecognized code mid-ladder with a token in hand and asserts it is terminal. The wired path from the argument to the resume input had no coverage; only the seam was tested. A scenario now goes through the function clap dispatches to, so dropping the argument turns it red. --- crates/gl/src/ipfs_cmd.rs | 172 +++++++++++++++++++++++++++++++++----- 1 file changed, 151 insertions(+), 21 deletions(-) diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index 6d2b2b72..1dd9f204 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -37,13 +37,22 @@ pub enum IpfsCmd { /// Identity directory (default: ~/.gitlawb) #[arg(long)] dir: Option, + /// Resume token from a scan that stopped at a bound, as printed by a + /// previous run that gave up with the result incomplete + #[arg(long, value_name = "TOKEN")] + scan: Option, }, } pub async fn run(args: IpfsArgs) -> Result<()> { match args.cmd { IpfsCmd::List { node, dir } => cmd_list(node, dir).await, - IpfsCmd::Get { cid, node, dir } => cmd_get(cid, node, dir).await, + IpfsCmd::Get { + cid, + node, + dir, + scan, + } => cmd_get(cid, node, dir, scan).await, } } @@ -120,8 +129,13 @@ fn diag(msg: &str) { tests::record_diag(msg); } -async fn cmd_get(cid: String, node: String, dir: Option) -> Result<()> { - cmd_get_inner(cid, node, dir, None, SCAN_DEADLINE, MAX_SCAN_RESUMES).await +async fn cmd_get( + cid: String, + node: String, + dir: Option, + scan: Option, +) -> Result<()> { + cmd_get_inner(cid, node, dir, scan, SCAN_DEADLINE, MAX_SCAN_RESUMES).await } /// The body of `gl ipfs get`, with the bounds and the starting continuation as @@ -171,9 +185,15 @@ async fn cmd_get_inner( }; // One deadline for the whole ladder, captured before the first request and used - // both as the loop's bound and as each request's own timeout, so the composed - // worst case is the deadline plus one clamped wait rather than the deadline plus - // the client's blanket 30s. + // both as the loop's bound and as each attempt's own timeout, so no attempt can + // start just under the deadline and then run a fresh unbounded 30s of its own. + // That wrap covers `get_authed` only, which resolves on the response HEADERS: the + // deadline is here to stop a slow legacy SEARCH, and extending it over the body + // read would abort a legitimate large download whose bytes are already flowing. + // The composed bounds that follow: headers by the deadline, then a body read under + // the client's blanket 30s, so a stalled body gives deadline + 30s; and on the + // give-up path a final clamped wait can overshoot the deadline by the clamp before + // the next check ends it, which is never followed by a body read. let start = tokio::time::Instant::now(); let mut requests = 0usize; loop { @@ -613,6 +633,7 @@ mod tests { "bafkreitestcid".to_string(), server.url(), Some(keystore.path().to_path_buf()), + None, ) .await .expect("signed get of a resolvable object should succeed"); @@ -635,7 +656,7 @@ mod tests { .create_async() .await; - let err = cmd_get("bafkreidenied".to_string(), server.url(), None) + let err = cmd_get("bafkreidenied".to_string(), server.url(), None, None) .await .expect_err("a 404 denial must be an error, not masked success"); assert!( @@ -683,7 +704,7 @@ mod tests { .create_async() .await; - cmd_get("bafkreiresume".to_string(), server.url(), None) + cmd_get("bafkreiresume".to_string(), server.url(), None, None) .await .expect("a search_incomplete 503 carrying a continuation must resume, not bail"); @@ -716,7 +737,7 @@ mod tests { .create_async() .await; - let err = cmd_get("bafkreicap".to_string(), server.url(), None) + let err = cmd_get("bafkreicap".to_string(), server.url(), None, None) .await .expect_err("a ladder that never completes must end in an error"); let told = told(&err); @@ -776,7 +797,7 @@ mod tests { .create_async() .await; - let err = cmd_get("bafkreiwedged".to_string(), server.url(), None) + let err = cmd_get("bafkreiwedged".to_string(), server.url(), None, None) .await .expect_err("a wedged ladder must end in an error"); let told = told(&err); @@ -816,7 +837,7 @@ mod tests { .create_async() .await; - let err = cmd_get("bafkreinotoken".to_string(), server.url(), None) + let err = cmd_get("bafkreinotoken".to_string(), server.url(), None, None) .await .expect_err("a truncation with no continuation must be an error"); let told = told(&err); @@ -860,7 +881,7 @@ mod tests { .create_async() .await; - let err = cmd_get("bafkreioverload".to_string(), server.url(), None) + let err = cmd_get("bafkreioverload".to_string(), server.url(), None, None) .await .expect_err("a first-request overload must be an error"); let told = told(&err); @@ -919,6 +940,7 @@ mod tests { "bafkreisigned".to_string(), server.url(), Some(keystore.path().to_path_buf()), + None, ) .await .expect("the resumed request must be signed and served"); @@ -943,7 +965,7 @@ mod tests { .create_async() .await; - let err = cmd_get("bafkreihostilebody".to_string(), server.url(), None) + let err = cmd_get("bafkreihostilebody".to_string(), server.url(), None, None) .await .expect_err("a hostile truncation body must still be an error"); let told = told(&err); @@ -987,7 +1009,7 @@ mod tests { .create_async() .await; - let err = cmd_get("bafkreihostiletoken".to_string(), server.url(), None) + let err = cmd_get("bafkreihostiletoken".to_string(), server.url(), None, None) .await .expect_err("a malformed continuation must be terminal"); let told = told(&err); @@ -1043,7 +1065,7 @@ mod tests { .create_async() .await; - let err = cmd_get("bafkreithrottled".to_string(), server.url(), None) + let err = cmd_get("bafkreithrottled".to_string(), server.url(), None, None) .await .expect_err("a mid-ladder 429 must be an error"); let told = told(&err); @@ -1117,7 +1139,7 @@ mod tests { .create_async() .await; - cmd_get("bafkreimidoverload".to_string(), server.url(), None) + cmd_get("bafkreimidoverload".to_string(), server.url(), None, None) .await .expect("a mid-ladder overload must be retried on the held token, not terminal"); @@ -1126,9 +1148,75 @@ mod tests { m3.assert_async().await; } + /// Scenario 10b. The classification default arm with a token ALREADY HELD. Every + /// other fixture reaches that arm token-less (scenario 5's first-request overload) + /// or never reaches it at all (scenario 9's 429 short-circuits on the status), so + /// the arm's terminality was certified only for the case where there was nothing + /// to resume with anyway. Here an unknown code arrives mid-ladder on a 500, which + /// is not the overload status, with a valid token in hand: it must still be + /// terminal. Two calls, and the fixture stops well short of the cap so a + /// misclassified retry shows up as a call count rather than a cap give-up. + #[tokio::test] + async fn test_cmd_get_mid_ladder_unknown_code_is_terminal_with_token_held() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + let calls = Arc::new(AtomicUsize::new(0)); + let c1 = calls.clone(); + let c2 = calls.clone(); + + let m1 = server + .mock("GET", "/ipfs/bafkreiunknowncode") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body_from_request({ + let t = t.clone(); + move |_req| { + c1.fetch_add(1, Ordering::SeqCst); + incomplete_body(Some(&t), "scan truncated").into_bytes() + } + }) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", "/ipfs/bafkreiunknowncode") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(500) + .with_header("content-type", "application/json") + .with_body_from_request(move |_req| { + c2.fetch_add(1, Ordering::SeqCst); + br#"{"error":"index_corrupt","message":"scan index unreadable"}"#.to_vec() + }) + .expect(1) + .create_async() + .await; + + let err = cmd_get("bafkreiunknowncode".to_string(), server.url(), None, None) + .await + .expect_err("an unknown code mid-ladder must be an error"); + let told = told(&err); + + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "an unrecognized code with a token held must be terminal, not retried" + ); + assert!( + told.contains("500"), + "the terminal must name the status, got: {told}" + ); + + m1.assert_async().await; + m2.assert_async().await; + } + /// Scenario 11. The wall-clock deadline bounds the whole loop, and it bounds each - /// request's own timeout, so the composed worst case is the deadline plus one - /// clamped wait. Injected through the seam because the shipped 60s is unreachable + /// attempt's own timeout. This drives the give-up tail, where the ladder never + /// reaches a body read, so its bound is the deadline plus one clamped wait (a + /// stalled body composes differently; see the note in `cmd_get_inner`). Injected + /// through the seam because the shipped 60s is unreachable /// under the 5s clamp and 8 resumes. #[tokio::test] async fn test_cmd_get_resume_ladder_stops_at_wall_clock_deadline() { @@ -1156,7 +1244,7 @@ mod tests { let started = Instant::now(); let err = cmd_get_inner( - "bafkreideadline".to_string(), + "bafkreislowscan".to_string(), server.url(), None, None, @@ -1217,7 +1305,7 @@ mod tests { .create_async() .await; - cmd_get("bafkreibound".to_string(), server.url(), None) + cmd_get("bafkreibound".to_string(), server.url(), None, None) .await .expect("a 2048-character token is within the bound and must be resumed with"); @@ -1247,7 +1335,7 @@ mod tests { .create_async() .await; - let err = cmd_get("bafkreioverbound".to_string(), server.url(), None) + let err = cmd_get("bafkreioverbound".to_string(), server.url(), None, None) .await .expect_err("an over-bound token must be terminal"); let told = told(&err); @@ -1305,6 +1393,47 @@ mod tests { res.expect("a supplied continuation must be used, not ignored"); } + /// R21, the wired half. The scenario above drives `cmd_get_inner` directly, so + /// it proves the resume INPUT works but says nothing about the `--scan` arg + /// reaching it. This one goes through `cmd_get`, the function clap dispatches + /// to, so a rewiring that drops the argument on the floor turns it red. Without + /// it the flag can be silently disconnected while every other resume test stays + /// green, and the invocation this command prints at a bound would be advice the + /// binary does not honor. + #[tokio::test] + async fn test_cmd_get_passes_the_scan_arg_through_to_the_resume_input() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t14"); + + let front = server + .mock("GET", "/ipfs/bafkreiwired") + .expect(0) + .create_async() + .await; + let resumed = server + .mock("GET", "/ipfs/bafkreiwired") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_body("object bytes") + .expect(1) + .create_async() + .await; + + let res = cmd_get( + "bafkreiwired".to_string(), + server.url(), + None, + Some(t.clone()), + ) + .await; + + front.assert_async().await; + resumed.assert_async().await; + res.expect("the --scan argument must reach the resume input through cmd_get"); + } + /// #173 review (F1): a base64 CID (multibase prefix 'm') can contain '/', '+', /// and '='. The client must percent-encode it into ONE path segment before /// building and signing `/ipfs/`; otherwise the '/' splits the target so @@ -1363,6 +1492,7 @@ mod tests { "bafkreitestcid".to_string(), server.url(), Some(empty.path().to_path_buf()), + None, ) .await .expect_err("an explicit --dir that fails to load must be an error"); From eac4d43c67d6c334f4dd23be00df0b83c2953175 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:11:59 -0500 Subject: [PATCH 62/77] docs(gl): describe how gl ipfs get resumes a truncated scan, and its limits Document the behaviour rather than the mechanism: that an object pinned before the node recorded its repo is found by scanning the inventory, that the scan stops at the node's ceilings, and that the command follows the resume token the node hands back instead of reporting a false not-found. State the bounds in values, not constant names, and state honestly what each one covers. The deadline bounds the search: every attempt gets the time left on it to produce response headers, and it deliberately does not cover downloading an object once found, so a large blob already streaming is never cut off. The error-body read runs outside it under the client's own timeout, which is why the longest a single run can take is about 95 seconds rather than the flat deadline, and the in-code comment now derives that instead of asserting it. Say what is not promised, too. A rate limit ends the ladder because its window is an hour and its Retry-After cannot be honored inside one invocation, and the node's per-IP fanout brake can end a ladder well short of the cap, so automatic resumption is not a guarantee that the object will be reached. Where a bound stops the ladder, the printed invocation is the only thing that makes progress: a bare re-run restarts at the first row, reproduces the same truncation and spends the per-IP budget again. --- README.md | 42 ++++++++++++++++++++++++++++++++++++ crates/gl/src/ipfs_cmd.rs | 45 +++++++++++++++++++++++++++++++++++---- 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2377f756..eda3b71f 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,48 @@ Notes: sends the bearer token **only** to your configured origin, never to a URL a node advertises, so a hostile node can't capture the key or redirect the solve. +### Fetching an object by CID + +```bash +gl ipfs list # CIDs this node has pinned +gl ipfs get bafkrei... > object.bin # object bytes on stdout +``` + +Objects that were pinned before the node started recording which repo they came +from are found by scanning its repo inventory, and that scan stops at the +per-request ceilings in the [Configuration](#configuration) table. A stopped scan +answers 503 with a resume token instead of a false "not found", and `gl ipfs get` +follows the token automatically: up to 8 resumes after the first request, so at +most 9 calls to the node, waiting between attempts for as long as the node's +`Retry-After` asks and never longer than 5 seconds. + +The whole ladder runs under a 60 second wall-clock deadline. The deadline bounds +the search, not the download: each attempt gets the time left on it to produce +response headers, and once an object is found its bytes stream under the client's +own 30 second HTTP timeout, so a large blob is never cut off part way. Adding the +one clamped wait that can still fire before the give-up check, a single run tops +out around 95 seconds. + +Two node-side brakes end a ladder early and are reported rather than retried +around. A 429 is terminal, because the node's rate-limit window is an hour and +that wait cannot be honored inside one invocation; a transient overload (a 503 +carrying no incomplete-scan code) is retried on the token already held, under the +same cap, clamp and deadline. The per-IP fanout brake can also stop a ladder well +short of the 9 calls, so automatic resumption is not a guarantee of reaching the +object. + +When a bound stops the ladder with a usable token in hand, the command prints the +token and the invocation that continues from it before exiting nonzero: + +```txt +resume from where this stopped: gl ipfs get bafkrei... --scan +``` + +Run that to carry on from where the scan stopped. Re-running without `--scan` +restarts at the first row, reproduces the same truncation and spends the node's +per-IP budget again, so the token is the only thing that makes progress. Tokens +are valid for an hour. + --- ## Architecture diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index 1dd9f204..b7e79f34 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -29,6 +29,38 @@ pub enum IpfsCmd { dir: Option, }, /// Retrieve and display a git object from the node by its CIDv1 + /// + /// Object bytes go to stdout so the command pipes; diagnostics go to stderr. + /// + /// Objects pinned before the node started recording which repo they came from + /// are found by scanning its repo inventory, and that scan stops at the node's + /// per-request ceilings. When it stops the node answers 503 with a resume token + /// rather than a false "not found", and this command follows it automatically: + /// up to 8 resumes after the first request, so at most 9 calls to the node, + /// waiting between attempts for as long as the node's Retry-After asks and never + /// longer than 5 seconds. + /// + /// The whole ladder runs under a 60 second wall-clock deadline. That deadline + /// bounds the search: each attempt gets only the time left on it to produce + /// response headers, and it deliberately does not cover the download of an + /// object once found, so a large blob already streaming is never cut off part + /// way. After the headers the transfer runs under the client's 30 second HTTP + /// timeout, and one final wait before the give-up check can add up to 5 seconds + /// more, so the longest a single run can take is about 95 seconds. + /// + /// A 429 ends the ladder immediately: the node's rate-limit window is an hour, + /// so the wait it asks for cannot be honored inside one invocation. A transient + /// overload (a 503 that carries no incomplete-scan code) is retried on the token + /// already held, under the same cap, clamp and deadline. The node's per-IP + /// fanout brake can also end a ladder well short of the cap, so automatic + /// resumption is not a guarantee that the object will be reached. + /// + /// Whenever one of those bounds stops the ladder with a usable token still in + /// hand, the command prints the token and the exact invocation that continues + /// from it, `gl ipfs get --scan `, and exits nonzero. Re-running + /// without the token restarts the scan at the first row, reproduces the same + /// truncation and spends the node's per-IP budget again, so the token is the + /// only thing that makes progress. Tokens are valid for an hour. Get { /// The CIDv1 string (e.g. bafkrei...) cid: String, @@ -190,10 +222,15 @@ async fn cmd_get_inner( // That wrap covers `get_authed` only, which resolves on the response HEADERS: the // deadline is here to stop a slow legacy SEARCH, and extending it over the body // read would abort a legitimate large download whose bytes are already flowing. - // The composed bounds that follow: headers by the deadline, then a body read under - // the client's blanket 30s, so a stalled body gives deadline + 30s; and on the - // give-up path a final clamped wait can overshoot the deadline by the clamp before - // the next check ends it, which is never followed by a body read. + // The composed bound that follows. The last attempt of any run starts strictly + // before the deadline, since both checks above run first, and reqwest's blanket + // 30s covers that whole request from its start through the end of its body, so it + // is over by deadline + 30s. Two reads sit under that 30s and not under the + // deadline: `write_object`'s success read, which ends the run, and + // `read_body_capped`'s error read, which on a retryable arm is followed by one + // clamped wait before the next iteration's check ends the loop. So the worst case + // is deadline + 30s + clamp, about 95s at the shipped defaults, and the give-up + // sleep is not a separate tail but the last term of that one. let start = tokio::time::Instant::now(); let mut requests = 0usize; loop { From 8c39cfdcb347ca824cbec7d6b56720df9517a472 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:09:45 -0500 Subject: [PATCH 63/77] fix(review): keep the resume token on every terminal that still holds one Review of the scan-continuation work found the ladder surfacing its token on three terminals and dropping it on three others: a mid-ladder 5xx that is not an overload, a truncation whose offered continuation fails validation while a good token is still held, and a transport error. In each case the user is told to start over, and a bare re-run restarts at the first row and re-spends the node's per-IP budget, which is the failure the token exists to prevent. It is surfaced on those paths now, still never on a definitive 404 or on the node's deliberate scan-wrapped signal, and a rejected token is still never echoed back. A body cut at the read cap used to change the answer. Truncated mid-JSON it failed to parse, lost its error code, and fell to the transient-overload arm, which retried the old position for every remaining attempt and then reported that the object might lie beyond the rows scanned when nothing past the first page had been. The cap now reports that it cut, and a cut body is terminal. Three guards were doing nothing. Mutation testing showed the Retry-After clamp could be deleted with every test still green, so a node advertising a day-long wait would have parked the client inside a single sleep that neither the attempt cap nor the deadline can interrupt. The rejection of a malformed caller-supplied token was equally untested, though the same property is proven on the node-offered side. The withheld-blob ladder asserted only that each rung was not a success, which any early exit satisfies without the rule ever being evaluated; it now witnesses that the scan reached the end of the table. The inter-attempt wait is bounded by the time left on the deadline, so a run can no longer overshoot the bound it advertises. That drops the wait out of the worst case entirely rather than shrinking it: the last attempt starts before the deadline and the client's total timeout covers that request through the end of its body, so a run is over by the deadline plus that timeout. The docs said about 95 seconds and now say about 90. They also said a large download already streaming is never cut off part way. That was wrong. The scan deadline does not cover the download, but the client's timeout is a total request deadline running until the body finishes, so a slow transfer is cut off. Both prose sites now say what the code does. Also: stdout is flushed before the command returns, so a closed pipe fails loudly instead of yielding a truncated object with a success status. --- README.md | 11 +- crates/gitlawb-node/src/api/ipfs.rs | 52 +++- crates/gl/src/http.rs | 16 +- crates/gl/src/ipfs_cmd.rs | 356 ++++++++++++++++++++++++++-- crates/gl/src/peer.rs | 4 +- crates/gl/src/sync.rs | 4 +- 6 files changed, 406 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index eda3b71f..de85a271 100644 --- a/README.md +++ b/README.md @@ -241,10 +241,13 @@ most 9 calls to the node, waiting between attempts for as long as the node's The whole ladder runs under a 60 second wall-clock deadline. The deadline bounds the search, not the download: each attempt gets the time left on it to produce -response headers, and once an object is found its bytes stream under the client's -own 30 second HTTP timeout, so a large blob is never cut off part way. Adding the -one clamped wait that can still fire before the give-up check, a single run tops -out around 95 seconds. +response headers, and once an object is found its bytes stream outside that +deadline. They are not unbounded, though. The client's own 30 second HTTP timeout +is a total request timeout, running from the start of a request until its body +has finished, so a transfer still going 30 seconds after its request began is cut +off. Waits between attempts never run past the deadline either, so a single run +tops out around 90 seconds: the deadline plus the 30 second timeout covering the +last attempt. Two node-side brakes end a ladder early and are reported rather than retried around. A 429 is terminal, because the node's rate-limit window is an hour and diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f2abb536..2e3b155d 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -265,8 +265,9 @@ impl LegacyScanPager { "legacy scan LIMIT must ask for at least one row" ); // Record the DB-facing ask before it is made. It sits above the timeout opener - // because the INV-22 guard reads a fixed lookback from the query call for that - // wrapper, and anything inserted inside the window eats its margin. + // because the committed guard that checks this query is deadline-wrapped reads a + // fixed lookback from the query call, and anything inserted inside that window + // eats its margin. #[cfg(test)] note_scan_limit(limit); let page = match tokio::time::timeout( @@ -4289,7 +4290,19 @@ mod tests { ); // The withheld blob gets its OWN full ladder, and is denied on every rung. + // + // A not-served assertion alone is satisfied by a ladder that never reached the + // boundary row: the per-IP work limiter ends one with a shed that carries no + // continuation, and so does an early taint, and a bare `None => break` cannot + // tell either from an honest exhaustion. So this half also witnesses HOW the + // ladder ended: every intermediate rung is specifically the retryable 503, and + // the last one is the `scan-wrapped` taint, which only a resumed scan that + // reached the END of the table emits. That is what makes "the rules withheld + // it" the reading. (Not a 404: a resumed scan has proven absence only over + // `[token, end)`, so the node deliberately withholds the definitive 404 and + // answers 503 with no continuation instead.) let mut token: Option = None; + let mut exhausted_at = None; for step in 1..=bound { let (status, body) = status_and_body( router @@ -4307,10 +4320,41 @@ mod tests { reaches it as the last row of a shortened page (step {step}): {body}" ); match continuation_of(&body) { - Some(t) => token = Some(t), - None => break, + Some(t) => { + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung of the withheld ladder is the retryable \ + 503 (step {step}): {body}" + ); + token = Some(t); + } + None => { + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the withheld ladder's last rung is the retryable 503 (step \ + {step}): {body}" + ); + assert!( + body["message"] + .as_str() + .unwrap_or_default() + .contains("scan-wrapped"), + "the withheld ladder must end by EXHAUSTING the inventory, which \ + only the scan-wrapped taint witnesses, not by a per-IP brake \ + that stopped it short of the boundary row (step {step}): {body}" + ); + exhausted_at = Some(step); + break; + } } } + assert!( + exhausted_at.is_some(), + "the withheld ladder must reach the end of the inventory within {bound} \ + rungs, otherwise no rung ever evaluated the rule that withholds the blob" + ); } /// The positive control for the fixture above: the identical inventory with the diff --git a/crates/gl/src/http.rs b/crates/gl/src/http.rs index 4a51dc45..a51facd7 100644 --- a/crates/gl/src/http.rs +++ b/crates/gl/src/http.rs @@ -206,21 +206,33 @@ async fn obtain_proof(cfg: IcaptchaCfg) -> Result { /// Read at most `cap` bytes of a response body. Bounds the allocation from a /// hostile or broken node returning a huge error body — the display is capped /// separately, but the read itself must not be unbounded (INV-6, read half). -pub(crate) async fn read_body_capped(mut resp: reqwest::Response, cap: usize) -> String { +/// +/// The second element of the return is whether the cap cut the body short. A +/// caller that CLASSIFIES on the body needs it: a cut body fails JSON parse, and +/// a parse failure is indistinguishable from a node that sent no code at all, so +/// without this flag an oversized body silently picks a different arm. +pub(crate) async fn read_body_capped(mut resp: reqwest::Response, cap: usize) -> (String, bool) { let mut buf: Vec = Vec::new(); + let mut truncated = false; while buf.len() < cap { match resp.chunk().await { Ok(Some(chunk)) => { let take = (cap - buf.len()).min(chunk.len()); buf.extend_from_slice(&chunk[..take]); if take < chunk.len() { + truncated = true; break; // hit the cap mid-chunk } } _ => break, // end of body or read error — return what we have } } - String::from_utf8_lossy(&buf).into_owned() + // A body that lands exactly on the cap may or may not have more behind it; + // report it as cut, since the classification that follows cannot tell either. + if buf.len() >= cap { + truncated = true; + } + (String::from_utf8_lossy(&buf).into_owned(), truncated) } /// Strip terminal-dangerous characters from (and cap the length of) a diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index b7e79f34..c264997b 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -43,10 +43,13 @@ pub enum IpfsCmd { /// The whole ladder runs under a 60 second wall-clock deadline. That deadline /// bounds the search: each attempt gets only the time left on it to produce /// response headers, and it deliberately does not cover the download of an - /// object once found, so a large blob already streaming is never cut off part - /// way. After the headers the transfer runs under the client's 30 second HTTP - /// timeout, and one final wait before the give-up check can add up to 5 seconds - /// more, so the longest a single run can take is about 95 seconds. + /// object once found. The download is not unbounded, though. The client's 30 + /// second HTTP timeout is a TOTAL request timeout, running from the moment a + /// request starts connecting until its body has finished, so a transfer still + /// going 30 seconds after its own request began is cut off. Waits between + /// attempts are bounded by the time left on the deadline as well as by the 5 + /// second clamp, so the longest a single run can take is about 90 seconds: the + /// deadline, plus the 30 second timeout covering the last attempt. /// /// A 429 ends the ladder immediately: the node's rate-limit window is an hour, /// so the wait it asks for cannot be honored inside one invocation. A transient @@ -222,15 +225,17 @@ async fn cmd_get_inner( // That wrap covers `get_authed` only, which resolves on the response HEADERS: the // deadline is here to stop a slow legacy SEARCH, and extending it over the body // read would abort a legitimate large download whose bytes are already flowing. + // reqwest's blanket 30s is what bounds the download instead; it is a TOTAL request + // timeout, from the start of the request through the end of its body, so a transfer + // slower than that from its own request's start IS cut off. // The composed bound that follows. The last attempt of any run starts strictly - // before the deadline, since both checks above run first, and reqwest's blanket - // 30s covers that whole request from its start through the end of its body, so it - // is over by deadline + 30s. Two reads sit under that 30s and not under the - // deadline: `write_object`'s success read, which ends the run, and - // `read_body_capped`'s error read, which on a retryable arm is followed by one - // clamped wait before the next iteration's check ends the loop. So the worst case - // is deadline + 30s + clamp, about 95s at the shipped defaults, and the give-up - // sleep is not a separate tail but the last term of that one. + // before the deadline, since both checks above run first, and that same blanket 30s + // covers its whole request, so it is over by deadline + 30s. Two reads sit under the + // 30s and not under the deadline: `write_object`'s success read, which ends the run, + // and `read_body_capped`'s error read, which on a retryable arm is followed by one + // wait. That wait adds no term of its own, because it is bounded by the time LEFT on + // the deadline as well as by the clamp. So the worst case is deadline + 30s, about + // 90s at the shipped defaults. let start = tokio::time::Instant::now(); let mut requests = 0usize; loop { @@ -258,7 +263,15 @@ async fn cmd_get_inner( None => format!("/ipfs/{encoded_cid}"), }; let resp = match tokio::time::timeout(remaining, client.get_authed(&path)).await { - Ok(r) => r.with_context(|| format!("failed to fetch CID {cid} from {node}"))?, + Ok(Ok(r)) => r, + Ok(Err(e)) => { + // A transport failure ends the ladder with the held token still + // pointing at a real position, so hand it back before propagating: + // otherwise a connection reset mid-ladder loses the only thing that + // makes progress on a re-run. + surface_resume(&cid, token.as_deref()); + return Err(e).with_context(|| format!("failed to fetch CID {cid} from {node}")); + } Err(_) => return Err(deadline_reached(&cid, token.as_deref(), deadline)), }; requests += 1; @@ -281,7 +294,7 @@ async fn cmd_get_inner( } let retry_after = parse_retry_after(resp.headers()); - let raw = read_body_capped(resp, 8 * 1024).await; + let (raw, truncated) = read_body_capped(resp, 8 * 1024).await; let parsed = serde_json::from_str::(&raw).ok(); let code = parsed.as_ref().and_then(|v| v["error"].as_str()); let node_msg = parsed @@ -300,7 +313,15 @@ async fn cmd_get_inner( // end and asks the caller back shortly, so the ladder continues on the token // it already holds. With no token there is nothing to resume, which falls to // the default arm below. - Some(_) | None if status == reqwest::StatusCode::SERVICE_UNAVAILABLE => token.clone(), + // + // A body the cap CUT SHORT is excluded from this arm. A cut body cannot + // parse, so its code reads as absent and an oversized `search_incomplete` + // would land here and be retried on the OLD token, replaying one position + // for every rung while the fresh continuation it offered goes unread. + // Unclassifiable is terminal, like any unrecognized code. + Some(_) | None if status == reqwest::StatusCode::SERVICE_UNAVAILABLE && !truncated => { + token.clone() + } _ => None, }; @@ -310,19 +331,36 @@ async fn cmd_get_inner( // Naming the bound, never echoing the value: a rejected token is // node-chosen text and has no business in a terminal message. let why = if offered.is_some() { + // The OFFERED token is unusable, but the one already held still + // points at a real position, so the ladder ends with something to + // resume from. Surface ours, never theirs. + surface_resume(&cid, token.as_deref()); format!( "the continuation it offered is not a resume token \ (expected 1 to {MAX_CONTINUATION_LEN} base64url characters)" ) } else { + // No continuation at all is the node's deliberate "the scan wrapped + // and finished" signal, so a resume hint here would invite a re-run + // that cannot find more than this one did. "it offered no continuation token".to_string() }; anyhow::bail!("node returned {status} with the scan incomplete and {why}: {msg}"); } + // Anything else stops the ladder with the held token still usable, so hand + // it back. The exception is a definitive 404: that is an answer, and a + // resume hint beside it would contradict it. + if status != reqwest::StatusCode::NOT_FOUND { + surface_resume(&cid, token.as_deref()); + } anyhow::bail!("node returned {status}: {msg}"); }; - tokio::time::sleep(retry_after.min(MAX_RETRY_AFTER)).await; + // Bounded three ways, and the deadline is the term that stops the give-up from + // overshooting: the loop only re-checks it at the top, so a wait longer than + // what is left would run past the deadline before anything noticed. + let left = deadline.saturating_sub(start.elapsed()); + tokio::time::sleep(retry_after.min(MAX_RETRY_AFTER).min(left)).await; token = Some(next); } } @@ -346,9 +384,14 @@ async fn write_object(resp: reqwest::Response) -> Result<()> { let bytes = resp.bytes().await.context("failed to read response body")?; use std::io::Write; - std::io::stdout() - .write_all(&bytes) - .context("failed to write to stdout")?; + // Flush explicitly rather than leaving the tail to the process-exit flush, which + // discards its error: `gl ipfs get > object.bin` onto a full disk or a + // closed pipe would otherwise leave a TRUNCATED file behind exit status 0, and on + // a content-addressed fetch a silently short object is the worst possible answer. + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + out.write_all(&bytes).context("failed to write to stdout")?; + out.flush().context("failed to flush stdout")?; Ok(()) } @@ -790,8 +833,8 @@ mod tests { "the give-up must name the incomplete result, got: {told}" ); assert!( - told.contains('8'), - "the give-up must name the resume cap, got: {told}" + told.contains(&format!("after {MAX_SCAN_RESUMES} automatic resumes")), + "the give-up must name the resume cap in words, got: {told}" ); assert!( told.contains(&held), @@ -1244,6 +1287,86 @@ mod tests { told.contains("500"), "the terminal must name the status, got: {told}" ); + // Terminal is only half of it. The ladder stopped holding a token that still + // points at a real position, and without it the caller's only recourse is a + // bare re-run that restarts at row 0 and re-spends the per-IP budget. Every + // terminal that holds a usable token must hand it back. + assert!( + told.contains(&t) && told.contains(&format!("--scan {t}")), + "the still-held continuation and its resuming invocation must be surfaced \ + on a mid-ladder terminal, got: {told}" + ); + + m1.assert_async().await; + m2.assert_async().await; + } + + /// Scenario 10c, the other reachable terminal that holds a token: rung 1 offers a + /// valid continuation, rung 2 answers `search_incomplete` with a MALFORMED one. + /// + /// The offered token is unusable and must never be echoed, but the token the client + /// already HOLDS is untouched by that rejection and still points at where the scan + /// stopped, so it is what must come back. Distinct from scenario 4, where the node + /// offers nothing at all: that is its deliberate "the scan wrapped and finished" + /// signal and carries no resume hint. + #[tokio::test] + async fn test_cmd_get_rejected_offered_token_still_surfaces_the_held_one() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + let malformed = "abc#def&ghi"; + let calls = Arc::new(AtomicUsize::new(0)); + let c1 = calls.clone(); + let c2 = calls.clone(); + + let m1 = server + .mock("GET", "/ipfs/bafkreirejectedoffer") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body_from_request({ + let t = t.clone(); + move |_req| { + c1.fetch_add(1, Ordering::SeqCst); + incomplete_body(Some(&t), "scan truncated").into_bytes() + } + }) + .expect(1) + .create_async() + .await; + let m2 = server + .mock("GET", "/ipfs/bafkreirejectedoffer") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body_from_request(move |_req| { + c2.fetch_add(1, Ordering::SeqCst); + incomplete_body(Some(malformed), "scan truncated").into_bytes() + }) + .expect(1) + .create_async() + .await; + + let err = cmd_get("bafkreirejectedoffer".to_string(), server.url(), None, None) + .await + .expect_err("a malformed offered continuation must be terminal"); + let told = told(&err); + + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "a rejected offer is terminal, so exactly two node calls" + ); + assert!( + told.contains(&t) && told.contains(&format!("--scan {t}")), + "the still-held continuation and its resuming invocation must be surfaced, \ + got: {told}" + ); + assert!( + !told.contains("abc#") && !told.contains("def&ghi"), + "a rejected token must never be echoed into the message, got: {told}" + ); m1.assert_async().await; m2.assert_async().await; @@ -1299,13 +1422,20 @@ mod tests { "the give-up must name the deadline, not the cap, got: {told}" ); let calls = calls.load(Ordering::SeqCst); + // The lower bound is 1, not 2. What this scenario proves is that the DEADLINE, + // not the cap, is what ends the ladder, and one call satisfies that as well as + // three do; requiring a resume as well made the test depend on a loaded runner + // fitting two round trips inside 2.5s, which is the likeliest flake in the + // suite. That a valid continuation is actually resumed with is scenario 1's job. assert!( - (2..9).contains(&calls), + (1..9).contains(&calls), "the deadline must stop the ladder before the cap, made {calls} calls" ); assert!( elapsed < Duration::from_secs(9), - "composed worst case is the 2.5s deadline plus one 5s clamped wait, took {elapsed:?}" + "the ladder never reaches a body read here, and every wait is bounded by the \ + time left on the 2.5s deadline, so the run is over near the deadline itself; \ + took {elapsed:?}" ); assert!( told.contains(&held) && told.contains(&format!("--scan {held}")), @@ -1542,4 +1672,184 @@ mod tests { m.assert_async().await; } + + /// #173 review (F4): a caller-supplied `--scan` value clears the same bar as a + /// node-offered one, BEFORE any request is signed. Both existing caller-supplied + /// scenarios pass a valid token, so the reject arm of that match was uncovered and + /// deleting the check left every test green even though the identical property is + /// covered on the node-offered side. A malformed value must fail with no node call + /// at all, and the rejection names the bound rather than echoing the value. + #[tokio::test] + async fn test_cmd_get_rejects_a_malformed_caller_supplied_continuation() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let bad = "abc#def&ghi"; + + // Nothing may be sent: the value would otherwise reach a signed target. + let m = server + .mock("GET", mockito::Matcher::Any) + .expect(0) + .create_async() + .await; + + let err = cmd_get_inner( + "bafkreibadinput".to_string(), + server.url(), + None, + Some(bad.to_string()), + SCAN_DEADLINE, + MAX_SCAN_RESUMES, + ) + .await + .expect_err("a malformed --scan value must be rejected"); + let told = told(&err); + + assert!( + told.contains(&MAX_CONTINUATION_LEN.to_string()) + && told.to_lowercase().contains("base64url"), + "the rejection must name the bound, got: {told}" + ); + assert!( + !told.contains("abc#") && !told.contains("def&ghi"), + "a rejected value must never be echoed back, got: {told}" + ); + + m.assert_async().await; + } + + /// #173 review (F3): the `Retry-After` clamp must actually bind somewhere. Every + /// other retryable fixture answers `Retry-After: 0` or `1`, both already under the + /// 5 second clamp, and the one 3600 in the suite rides a 429 that returns before + /// the header is ever parsed. So deleting `.min(MAX_RETRY_AFTER)` left the whole + /// suite green. + /// + /// Here a retryable 503 asks for an hour, with a valid continuation, under a + /// deadline set a little wider than the clamp. Clamped, the first wait is 5 + /// seconds and the deadline still has room for a second attempt. Unclamped, that + /// one wait consumes the whole deadline and the run ends after a single call. The + /// call count is what separates them, and it fails fast rather than hanging, + /// because the wait is also bounded by the time left on the deadline. + #[tokio::test] + async fn test_cmd_get_clamps_a_hostile_retry_after_below_the_deadline() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let calls = Arc::new(AtomicUsize::new(0)); + let c = calls.clone(); + + let m = server + .mock("GET", mockito::Matcher::Any) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "3600") + .with_body_from_request(move |req| { + c.fetch_add(1, Ordering::SeqCst); + let next = next_token(scan_of(req.path_and_query()).as_deref()); + incomplete_body(Some(&next), "scan truncated").into_bytes() + }) + .expect_at_least(1) + .create_async() + .await; + + let started = Instant::now(); + let err = cmd_get_inner( + "bafkreihostileretry".to_string(), + server.url(), + None, + None, + Duration::from_secs(6), + MAX_SCAN_RESUMES, + ) + .await + .expect_err("a ladder that outruns the deadline must end in an error"); + let elapsed = started.elapsed(); + let calls = calls.load(Ordering::SeqCst); + + assert!( + calls >= 2, + "the clamp caps a single wait at {}s, well under the 6s deadline, so one \ + hostile Retry-After must not swallow the run: made {calls} calls", + MAX_RETRY_AFTER.as_secs() + ); + assert!( + elapsed < Duration::from_secs(12), + "every wait is bounded by the clamp and by the time left on the 6s \ + deadline, so the run ends near the deadline; took {elapsed:?}" + ); + assert!( + told(&err).to_lowercase().contains("deadline"), + "the give-up must name the deadline, got: {}", + told(&err) + ); + + m.assert_async().await; + } + + /// #173 review (F9): a `search_incomplete` body the 8 KiB read cap CUT SHORT must + /// be terminal, not retried. + /// + /// A cut body cannot parse, so its `error` code reads as absent, and on a 503 that + /// used to fall through to the generic overload arm, which resumes on the token + /// ALREADY HELD. The fresh continuation the node offered is inside the part that + /// was never read, so the ladder replays one position for every remaining rung: a + /// 9000-character body drove eight requests carrying the old token. Unclassifiable + /// is terminal, like any unrecognized code. + #[tokio::test] + async fn test_cmd_get_truncated_incomplete_body_is_terminal_not_a_replay() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + let calls = Arc::new(AtomicUsize::new(0)); + let c1 = calls.clone(); + let c2 = calls.clone(); + + let m1 = server + .mock("GET", "/ipfs/bafkreicutbody") + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body_from_request({ + let t = t.clone(); + move |_req| { + c1.fetch_add(1, Ordering::SeqCst); + incomplete_body(Some(&t), "scan truncated").into_bytes() + } + }) + .expect(1) + .create_async() + .await; + // Well past the 8 KiB cap, with the fresh continuation behind the cut. + let m2 = server + .mock("GET", "/ipfs/bafkreicutbody") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body_from_request(move |_req| { + c2.fetch_add(1, Ordering::SeqCst); + incomplete_body(Some(&make_token("t2")), &"x".repeat(9000)).into_bytes() + }) + .expect(1) + .create_async() + .await; + + let err = cmd_get("bafkreicutbody".to_string(), server.url(), None, None) + .await + .expect_err("an unclassifiable 503 body must be an error"); + let told = told(&err); + + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "a body cut by the read cap must end the ladder, not replay the held token \ + for every remaining rung" + ); + assert!( + told.contains(&t) && told.contains(&format!("--scan {t}")), + "the still-held continuation and its resuming invocation must be surfaced, \ + got: {told}" + ); + + m1.assert_async().await; + m2.assert_async().await; + } } diff --git a/crates/gl/src/peer.rs b/crates/gl/src/peer.rs index 6b55b882..c0f7bca8 100644 --- a/crates/gl/src/peer.rs +++ b/crates/gl/src/peer.rs @@ -199,7 +199,7 @@ async fn cmd_add(peer_url: String, node: String, dir: Option) -> Result // in the command: bound the read, and defang the message before it reaches // the terminal through the error return. An announce reply is a DID, a URL // and a count, so 8 KiB is well past what the shape needs. - let raw = read_body_capped(resp, 8 * 1024).await; + let (raw, _truncated) = read_body_capped(resp, 8 * 1024).await; if let Some(failure) = remote_announce_failure(status, &raw) { anyhow::bail!("{failure}"); @@ -237,7 +237,7 @@ async fn cmd_add(peer_url: String, node: String, dir: Option) -> Result // node (or a MITM on plain http) must not force an unbounded // read. A body that does not parse stays `Null`, which still // routes a non-success status to the warning. - let raw = read_body_capped(resp, 8 * 1024).await; + let (raw, _truncated) = read_body_capped(resp, 8 * 1024).await; let result: Value = serde_json::from_str(&raw).unwrap_or(Value::Null); let (is_warning, line) = local_add_report(status, &result); if is_warning { diff --git a/crates/gl/src/sync.rs b/crates/gl/src/sync.rs index c3ff7972..60950d99 100644 --- a/crates/gl/src/sync.rs +++ b/crates/gl/src/sync.rs @@ -46,7 +46,7 @@ pub async fn run(args: SyncArgs) -> Result<()> { if !status.is_success() { // Bound the read: a hostile or broken node must not force an // unbounded allocation just to surface a denial (INV-6, read half). - let raw = read_body_capped(resp, 8 * 1024).await; + let (raw, _truncated) = read_body_capped(resp, 8 * 1024).await; let msg = serde_json::from_str::(&raw) .ok() .and_then(|v| { @@ -261,7 +261,7 @@ mod tests { .create_async() .await; let resp = reqwest::get(format!("{}/big", server.url())).await.unwrap(); - let out = read_body_capped(resp, 8192).await; + let (out, _truncated) = read_body_capped(resp, 8192).await; assert!(out.len() <= 8192, "read not bounded: {} bytes", out.len()); assert!(!out.is_empty(), "expected some body"); } From de3f230940f21d88da48a972bddeb877420d2165 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:10:04 -0500 Subject: [PATCH 64/77] fix(gl): stop carrying request signatures to a host the node redirected us to The node client followed redirects with no policy, and reqwest strips only the classic credential headers when a redirect crosses origins. RFC 9421 signature headers are not on that list, so a node answering 302 could hand a caller's signature to another host, which then served whatever bytes it liked for a content address that never hashed to them. Following continuations made it worse by turning one signed request per invocation into as many as nine. The signature covers the method, the path and the body digest but not the authority, and the node accepts any signature inside a five minute skew with no nonce ledger, so one harvested header reads a caller's private paths at any node in the fleet. Redirects are now scoped to the origin that issued them. Same-origin ones still follow, so an http to https upgrade or a trailing-slash normalization keeps working, and a cross-origin one stops rather than errors, so the caller sees the 3xx through the status path it already has. A downgrade back to http on the same host is refused too: the origin matches, but the signature would leave in cleartext, which is the same leak by a slower route. A custom policy replaces reqwest's built-in chain limit, so the limit is restated rather than lost. The recovery path in clone had the same silence the CLI just lost: it skipped a blob on any non-success without saying why, so a truncated scan, a permission denial and a dead gateway were one indistinguishable outcome. It now names the status or the transport error, capped and stripped of control characters since the text comes from a caller-chosen gateway, and continues as before. Reading a response body no longer conflates three endings. The cap cutting the body, the body finishing, and the read failing partway were all one silent break, so a failed read surfaced as an error message that stopped after the colon. Each is now reported, and a body that could not be read says so. Coverage for three paths that were previously argued rather than run: a transport failure mid-ladder still hands back the resume token, a scan that ends before any ceiling still answers a definitive not-found, and a response whose body stalls after its headers is still ended by the client's own deadline. --- crates/gitlawb-node/src/api/ipfs.rs | 73 +++++ crates/gl/src/clone.rs | 187 ++++++++++++- crates/gl/src/http.rs | 405 +++++++++++++++++++++++++++- crates/gl/src/ipfs_cmd.rs | 149 +++++++++- crates/gl/src/peer.rs | 4 +- crates/gl/src/sync.rs | 4 +- 6 files changed, 806 insertions(+), 16 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 2e3b155d..00188ee7 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -3781,6 +3781,79 @@ mod tests { // scarce global walk permits for up to the whole request budget. // ---------------------------------------------------------------------------- + /// Scenario 0, the case every other scan test skips: a ceiling that EXCEEDS the + /// table, on a scan that was never resumed. This is the one path that still owes + /// the caller a definitive 404. + /// + /// The three fixtures above and beside it all park the holder PAST a ceiling, so + /// each one proves the truncating half: nothing unproven may answer 404. Nobody + /// was covering the converse, and it is the more dangerous direction to lose, + /// because a scan that quietly stops short and STILL answers 404 reports existing + /// content as absent. Here five rows sit under a ceiling of 64, the requested CID + /// genuinely resolves to nothing, and the scan runs off the end of the table with + /// its ceilings untouched. + /// + /// The counters are what make "ran to exhaustion" an observation rather than an + /// inference. `scan_rows` reaching all five says the walk covered the table, and + /// `scan_limit` at 8 says both asks went out at the full page size, so no budget + /// ever shortened one. A 404 with either counter short would be the false-absent + /// answer wearing the right status. + /// + /// MUTATION (RED): drop the `pager.resumed &&` guard on the wrapped-scan taint and + /// this exhausted scan taints as `scan-wrapped`, so the tail becomes a 503 and the + /// definitive answer is never reached. + #[sqlx::test] + async fn get_by_cid_unresumed_scan_under_the_ceiling_is_a_definitive_404(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 4; + // Well clear of the five rows below: the point is a ceiling that never binds. + state.ipfs_max_legacy_scan_rows = 64; + seed_root_denying_repos(&state, "underceiling", 5, 0).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + + let peer: SocketAddr = "203.0.113.160:5000".parse().unwrap(); + crate::api::ipfs::reset_scan_rows(); + crate::api::ipfs::reset_scan_limit(); + let (status, body) = status_and_body( + ipfs_router(state) + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + + let rows = crate::api::ipfs::scan_rows(); + let limit = crate::api::ipfs::scan_limit(); + assert_eq!( + rows, 5, + "the scan must reach every seeded row before it may call the object absent. \ + Read {rows} of 5" + ); + assert_eq!( + limit, 8, + "two asks at the full page size (4 + 4): with the ceiling far above the \ + table, nothing may shorten the query, and a shortened one would mean the \ + 404 below rested on a bounded walk. Asked for {limit}" + ); + assert_eq!( + status, + StatusCode::NOT_FOUND, + "a scan that ran off the end of the table with no ceiling touched and no \ + resume token has proven absence over the whole inventory, so the answer is \ + the definitive 404, not a truncation 503: {body}" + ); + assert!( + continuation_of(&body).is_none(), + "there is nothing left to resume, so a definitive 404 must carry no \ + continuation: {body}" + ); + assert_ne!( + body["error"], "search_incomplete", + "the answer is an absence, not a truncation: {body}" + ); + } + /// Scenario 1: an all-root-denied inventory stops at the row ceiling. /// /// Every seeded repo is private and the caller is anonymous, so each row is a root diff --git a/crates/gl/src/clone.rs b/crates/gl/src/clone.rs index 6754c5e0..fc90316d 100644 --- a/crates/gl/src/clone.rs +++ b/crates/gl/src/clone.rs @@ -12,9 +12,29 @@ use serde::Deserialize; use std::path::Path; use std::process::Command; -use crate::http::NodeClient; +use crate::http::{sanitize_node_msg, NodeClient}; use crate::identity::load_keypair_from_dir; +/// Report why one candidate blob could not be recovered, in the register of the +/// recovery loop's other warnings. +/// +/// Both halves are gateway-supplied text on their way to a terminal: the oid comes +/// out of a manifest fetched from a caller-chosen Arweave gateway, and a transport +/// error carries the URL the same manifest chose. So the whole line is defanged and +/// length-capped, like every other node/gateway string this crate prints. +/// +/// Under `cfg(test)` the line is also mirrored into a per-thread buffer, so a test +/// can assert the loop actually says what it skipped rather than only that it +/// returned nothing. +fn warn_skip(oid: &str, why: &str) { + let line = sanitize_node_msg(&format!( + "warning: could not fetch encrypted blob {oid}: {why}; skipping" + )); + eprintln!("{line}"); + #[cfg(test)] + tests::record_warn(&line); +} + #[derive(Args)] pub struct CloneArgs { /// Repo to clone: gitlawb:/// or /. @@ -684,7 +704,21 @@ async fn recover_from_arweave( } let env_resp = match client.get(format!("{ig}/ipfs/{cid}")).send().await { Ok(r) if r.status().is_success() => r, - _ => continue, + // Every other outcome used to leave through a bare `continue`, so a + // gateway that answered 503, 404, or nothing at all was reported to the + // caller as "blob not recoverable" with no reason attached. This is the + // second in-repo client of GET /ipfs/{cid} and it has no resume ladder, + // so saying what happened is all the recourse there is. It still skips to + // the next candidate either way: one unreachable blob must not end the + // recovery of the rest. + Ok(r) => { + warn_skip(&oid, &format!("gateway returned {}", r.status())); + continue; + } + Err(e) => { + warn_skip(&oid, &format!("gateway request failed: {e}")); + continue; + } }; let Ok(envelope) = env_resp.bytes().await else { continue; @@ -801,9 +835,32 @@ pub async fn run(args: CloneArgs) -> Result<()> { #[cfg(test)] mod tests { use super::*; + use std::cell::RefCell; use std::process::Command; use tempfile::TempDir; + thread_local! { + /// Test-visible copy of the stderr warnings `warn_skip` emits. + /// `#[tokio::test]` runs the future on the test's own thread, so a + /// thread-local is enough. + static WARNINGS: RefCell = const { RefCell::new(String::new()) }; + } + + pub(super) fn record_warn(msg: &str) { + WARNINGS.with(|w| { + w.borrow_mut().push_str(msg); + w.borrow_mut().push('\n'); + }); + } + + fn reset_warnings() { + WARNINGS.with(|w| w.borrow_mut().clear()); + } + + fn warnings() -> String { + WARNINGS.with(|w| w.borrow().clone()) + } + fn g(args: &[&str], dir: &Path) { assert!(Command::new("git") .args(args) @@ -1255,6 +1312,132 @@ mod tests { assert_eq!(a.get("o1").map(String::as_str), Some("cidTS")); } + /// A gateway that refuses one blob must SAY so, and must not take the rest of the + /// recovery down with it. + /// + /// The IPFS fetch used to leave through a bare `_ => continue` on every + /// non-success, so a 503 (the truncation status a tuned node makes more likely), + /// a 404, and a dead connection all reached the caller as the same silent "blob + /// not recoverable". Two withheld blobs here: the gateway answers 503 for the + /// first and serves the second. The recovered path proves the loop continued; the + /// warning proves the skip was reported and names both the object and the status. + #[tokio::test] + async fn recover_from_arweave_reports_a_refusing_gateway_and_continues() { + use gitlawb_core::encrypt::seal_blob; + use gitlawb_core::identity::Keypair; + + reset_warnings(); + let (td, url) = bare_remote(&[ + ("public/a.txt", b"pub\n"), + ("secret/b.txt", b"SECRET B\n"), + ("secret/c.txt", b"SECRET C\n"), + ]); + let dest = td.path().join("dest"); + let bare = url.strip_prefix("file://").unwrap(); + assert!(Command::new("git") + .args(["-C", bare, "config", "uploadpack.allowFilter", "true"]) + .status() + .unwrap() + .success()); + setup_partial_clone(&dest, &url, &["/secret/**".to_string()], &[], None).unwrap(); + + let oid_of = |path: &str| { + let out = Command::new("git") + .args([ + "-C", + dest.to_str().unwrap(), + "rev-parse", + &format!("HEAD:{path}"), + ]) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let refused_oid = oid_of("secret/b.txt"); + let served_oid = oid_of("secret/c.txt"); + + // Origin death, so recovery has to go through the gateways. + std::fs::remove_dir_all(bare).unwrap(); + + let reader = Keypair::generate(); + let envelope = seal_blob(b"SECRET C\n", &[reader.verifying_key()]).unwrap(); + + let refused_cid = "cidrefused"; + let served_cid = "cidserved"; + let mut server = mockito::Server::new_async().await; + let _gql = server + .mock("POST", "/graphql") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"data":{"transactions":{"edges":[{"node":{"id":"TX1"}}]}}}"#) + .create_async() + .await; + let manifest_body = serde_json::json!({ + "timestamp": "2026-06-11T00:00:00Z", + "blobs": [ + { "oid": refused_oid, "cid": refused_cid }, + { "oid": served_oid, "cid": served_cid }, + ], + }) + .to_string(); + let _tx = server + .mock("GET", "/TX1") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(manifest_body) + .create_async() + .await; + let refusal = server + .mock("GET", format!("/ipfs/{refused_cid}").as_str()) + .with_status(503) + .with_header("content-type", "application/json") + .with_body(r#"{"error":"search_incomplete","message":"scan truncated"}"#) + .expect(1) + .create_async() + .await; + let ok = server + .mock("GET", format!("/ipfs/{served_cid}").as_str()) + .with_status(200) + .with_body(envelope) + .expect(1) + .create_async() + .await; + + let paths = recover_from_arweave( + &server.url(), + &server.url(), + "alice", + "myrepo", + &dest, + &reader, + ) + .await + .unwrap(); + + refusal.assert_async().await; + ok.assert_async().await; + assert_eq!( + paths, + vec!["secret/c.txt".to_string()], + "a refused candidate must not stop the loop reaching the next one" + ); + + let warnings = warnings(); + assert!( + warnings.contains(&refused_oid), + "the skip must name the object it could not fetch, got: {warnings}" + ); + assert!( + warnings.contains("503"), + "the skip must name the gateway's status rather than failing silently, \ + got: {warnings}" + ); + assert!( + !warnings.contains(&served_oid), + "the blob that was served must not be reported as skipped, got: {warnings}" + ); + } + /// Read-path end-to-end over a mocked Arweave + IPFS gateway: discover the /// manifest via GraphQL, fetch it, fetch the envelope, decrypt with the /// caller's key, and install the previously-withheld blob. diff --git a/crates/gl/src/http.rs b/crates/gl/src/http.rs index a51facd7..c1f610a7 100644 --- a/crates/gl/src/http.rs +++ b/crates/gl/src/http.rs @@ -15,6 +15,63 @@ use icaptcha_client::IcaptchaCfg; /// (absorbs proof expiry / first-seen replay). const MAX_ICAPTCHA_RETRIES: usize = 2; +/// Total request timeout: from the start of connecting through the end of the +/// response body, so it bounds a slow download and not just a slow handshake. +const TOTAL_REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + +/// Longest redirect chain followed. `Policy::custom` replaces reqwest's built-in +/// limit, so the loop bound has to be restated here; the value is reqwest's own +/// default. Same-origin redirects can cycle, and without this a node answering 302 +/// to itself would spin forever. +const MAX_REDIRECTS: usize = 10; + +/// Follow a redirect only when it stays on the origin that issued it. +/// +/// Every request this client sends may carry RFC 9421 `Signature` and +/// `Signature-Input` headers, and reqwest strips only `Authorization`, `Cookie`, +/// `Proxy-Authorization` and `WWW-Authenticate` when a redirect crosses hosts. The +/// signature would survive, and it binds `@method`, `@path` and `content-digest` +/// with no authority component, so a node answering 302 could hand a working +/// credential to a host of its choosing and read as the caller anywhere for as long +/// as the node's clock-skew window lasts. +/// +/// `Policy::none()` would have been the simpler answer, but same-origin redirects +/// are legitimate here (a node fronted by a proxy that upgrades http to https, or +/// normalizes a trailing slash), so the policy is scoped to the origin rather than +/// switched off. Refusal is `stop`, not `error`: the 3xx comes back as an ordinary +/// response and each caller reports it through the status path it already has. +/// +/// Host and port must match exactly. Port is compared as `Url::port`, which is +/// `None` for a scheme's default port, so http -> https on the same host compares +/// equal while http -> http on a different port does not. A downgrade from https to +/// http is refused as well: the target is the same host, but the signature would go +/// out in cleartext, which is the same credential leak by a slower route. +fn same_origin_redirect(attempt: reqwest::redirect::Attempt<'_>) -> reqwest::redirect::Action { + let Some(previous) = attempt.previous().last() else { + // No previous URL to compare against. Unreachable through reqwest, which + // pushes the redirecting URL before consulting the policy, but the safe + // reading of "cannot prove same-origin" is to refuse. + return attempt.stop(); + }; + if attempt.previous().len() >= MAX_REDIRECTS { + return attempt.stop(); + } + if may_follow(previous, attempt.url()) { + attempt.follow() + } else { + attempt.stop() + } +} + +/// The decision itself, split out because `redirect::Attempt` cannot be built outside +/// reqwest, so this is the only way to run the scheme and port branches both ways +/// rather than reasoning about them. +fn may_follow(previous: &reqwest::Url, next: &reqwest::Url) -> bool { + let same_origin = next.host_str() == previous.host_str() && next.port() == previous.port(); + let downgraded = previous.scheme() == "https" && next.scheme() != "https"; + same_origin && !downgraded +} + pub struct NodeClient { inner: reqwest::Client, pub node_url: String, @@ -23,8 +80,20 @@ pub struct NodeClient { impl NodeClient { pub fn new(node_url: impl Into, keypair: Option) -> Self { + Self::with_timeout(node_url, keypair, TOTAL_REQUEST_TIMEOUT) + } + + /// `new` with the total request timeout as a parameter, so a test can drive the + /// timeout's behaviour without waiting out the shipped value. `new` supplies the + /// shipped one, which is what makes the scaled-down test cover the real client. + fn with_timeout( + node_url: impl Into, + keypair: Option, + timeout: std::time::Duration, + ) -> Self { let inner = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(30)) + .timeout(timeout) + .redirect(reqwest::redirect::Policy::custom(same_origin_redirect)) .user_agent(format!("gl/{} gitlawb-cli", env!("CARGO_PKG_VERSION"))) .build() .expect("failed to build HTTP client"); @@ -203,17 +272,37 @@ async fn obtain_proof(cfg: IcaptchaCfg) -> Result { .context("iCaptcha solver task panicked")? } +/// What a capped body read produced. `text` is the bytes that arrived; the two flags +/// say why the read stopped where it did, which a caller that CLASSIFIES on the body +/// cannot work out from the text alone. +pub(crate) struct CappedBody { + /// The bytes read, lossily decoded. + pub(crate) text: String, + /// The cap cut the body short. + pub(crate) truncated: bool, + /// A chunk read FAILED part-way through, so the body is not merely short, it is + /// unfinished and the node may have had more to say. + pub(crate) read_failed: bool, +} + /// Read at most `cap` bytes of a response body. Bounds the allocation from a /// hostile or broken node returning a huge error body — the display is capped /// separately, but the read itself must not be unbounded (INV-6, read half). /// -/// The second element of the return is whether the cap cut the body short. A -/// caller that CLASSIFIES on the body needs it: a cut body fails JSON parse, and -/// a parse failure is indistinguishable from a node that sent no code at all, so -/// without this flag an oversized body silently picks a different arm. -pub(crate) async fn read_body_capped(mut resp: reqwest::Response, cap: usize) -> (String, bool) { +/// `truncated` reports whether the cap cut the body short. A caller that CLASSIFIES +/// on the body needs it: a cut body fails JSON parse, and a parse failure is +/// indistinguishable from a node that sent no code at all, so without this flag an +/// oversized body silently picks a different arm. +/// +/// `read_failed` reports the other way a body can end early. A mid-body read error +/// used to leave through the same exit as a clean end of stream, so a 500 whose body +/// died in transit surfaced as an empty message and the caller was told +/// `node returned 500: ` with nothing after the colon. That is a report of what the +/// node said, and the node never got to say it. +pub(crate) async fn read_body_capped(mut resp: reqwest::Response, cap: usize) -> CappedBody { let mut buf: Vec = Vec::new(); let mut truncated = false; + let mut read_failed = false; while buf.len() < cap { match resp.chunk().await { Ok(Some(chunk)) => { @@ -224,7 +313,11 @@ pub(crate) async fn read_body_capped(mut resp: reqwest::Response, cap: usize) -> break; // hit the cap mid-chunk } } - _ => break, // end of body or read error — return what we have + Ok(None) => break, // clean end of body + Err(_) => { + read_failed = true; + break; + } } } // A body that lands exactly on the cap may or may not have more behind it; @@ -232,7 +325,11 @@ pub(crate) async fn read_body_capped(mut resp: reqwest::Response, cap: usize) -> if buf.len() >= cap { truncated = true; } - (String::from_utf8_lossy(&buf).into_owned(), truncated) + CappedBody { + text: String::from_utf8_lossy(&buf).into_owned(), + truncated, + read_failed, + } } /// Strip terminal-dangerous characters from (and cap the length of) a @@ -633,6 +730,298 @@ mod tests { ic.answer.assert(); } + // ── redirect policy ───────────────────────────────────────────────── + + /// The signed headers must not survive a redirect off the node's origin. + /// + /// reqwest strips only `Authorization`, `Cookie`, `Proxy-Authorization` and + /// `WWW-Authenticate` across hosts, so `Signature` and `Signature-Input` used to + /// ride a 302 straight to whatever origin the node named. The signature binds + /// `@method`, `@path` and `content-digest` and nothing about the authority, so the + /// receiving host holds a credential that reads path-scoped objects as the caller + /// at any node until the clock-skew window closes. + /// + /// Two mockito servers are two ports on one host, which is exactly the boundary + /// this policy draws (and the one reqwest's own header stripping draws). The + /// second server answers everything and expects nothing: a followed redirect + /// fails the expectation whether or not the signature came with it. MUTATION + /// (RED): drop the `.redirect(...)` line and the second server is hit. + #[tokio::test] + async fn cross_origin_redirect_is_not_followed() { + let mut elsewhere = Server::new_async().await; + let never = elsewhere + .mock("GET", mockito::Matcher::Any) + .with_status(200) + .with_body("bytes from the redirect target") + .expect(0) + .create_async() + .await; + let signature_seen = elsewhere + .mock("GET", mockito::Matcher::Any) + .match_header("signature", mockito::Matcher::Any) + .with_status(200) + .expect(0) + .create_async() + .await; + + let mut node = Server::new_async().await; + let bounce = node + .mock("GET", "/api/v1/thing") + .with_status(302) + .with_header("location", &format!("{}/api/v1/thing", elsewhere.url())) + .expect(1) + .create_async() + .await; + + let client = NodeClient::new(node.url(), Some(test_keypair())); + let resp = client.get_signed("/api/v1/thing").await.unwrap(); + + assert_eq!( + resp.status(), + 302, + "a refused redirect stops rather than errors, so the caller sees the 3xx \ + and reports it through the status path it already has" + ); + let body = resp.text().await.unwrap(); + assert!( + !body.contains("bytes from the redirect target"), + "the redirect target's bytes must never reach the caller, got: {body}" + ); + bounce.assert_async().await; + never.assert_async().await; + signature_seen.assert_async().await; + } + + /// Every branch of the decision, both ways. The end-to-end test above drives one + /// pair of http origins, which is all mockito can serve; the scheme cases and the + /// default-port equivalence have no other way to be run. + #[test] + fn may_follow_covers_each_origin_branch() { + let url = |s: &str| reqwest::Url::parse(s).unwrap(); + let cases: &[(&str, &str, bool, &str)] = &[ + ( + "http://node.example/a", + "http://node.example/b", + true, + "same origin, different path", + ), + ( + "http://node.example/a", + "https://node.example/a", + true, + "http to https on one host: both ports are the scheme default", + ), + ( + "https://node.example/a", + "https://node.example/a/", + true, + "trailing-slash normalization", + ), + ( + "https://node.example:8443/a", + "https://node.example:8443/b", + true, + "same explicit port", + ), + ( + "http://node.example/a", + "http://attacker.example/a", + false, + "different host", + ), + ( + "http://node.example/a", + "http://node.example:8080/a", + false, + "same host, different port", + ), + ( + "https://node.example/a", + "http://node.example/a", + false, + "https downgraded to cleartext on the same host", + ), + ( + "https://node.example/a", + "http://node.example:443/a", + false, + "a downgrade dressed up as the https port", + ), + ]; + for (previous, next, expected, why) in cases { + assert_eq!( + may_follow(&url(previous), &url(next)), + *expected, + "{previous} -> {next} ({why})" + ); + } + } + + /// The other direction: a same-origin redirect is still followed, so a node + /// fronted by a proxy that normalizes a path keeps working. Without this the + /// policy could be tightened to `Policy::none()` and nothing would notice. + #[tokio::test] + async fn same_origin_redirect_is_followed() { + let mut node = Server::new_async().await; + let bounce = node + .mock("GET", "/api/v1/thing") + .with_status(301) + .with_header("location", "/api/v1/thing/") + .expect(1) + .create_async() + .await; + let target = node + .mock("GET", "/api/v1/thing/") + .with_status(200) + .with_body("normalized") + .expect(1) + .create_async() + .await; + + let client = NodeClient::new(node.url(), Some(test_keypair())); + let resp = client.get_signed("/api/v1/thing").await.unwrap(); + + assert_eq!(resp.status(), 200); + assert_eq!(resp.text().await.unwrap(), "normalized"); + bounce.assert_async().await; + target.assert_async().await; + } + + // ── read_body_capped ──────────────────────────────────────────────── + + /// A body whose read FAILS mid-stream must be distinguishable from a body that + /// ended. Both used to leave through the same `_ => break`, so a 500 whose body + /// died in transit produced an empty message and the caller was told + /// `node returned 500: ` with nothing after the colon. + /// + /// The fixture is a raw listener that promises 64 bytes in `Content-Length`, + /// writes 5, and closes. mockito cannot express that: it always completes the + /// response it advertises. MUTATION (RED): restore the single `_ => break` arm + /// (or hard-code `read_failed: false`) and the flag reads false. + #[tokio::test] + async fn read_body_capped_flags_a_mid_body_read_failure() { + let addr = spawn_short_body_listener().await; + let resp = reqwest::get(format!("http://{addr}/truncated")) + .await + .expect("headers arrive before the body is cut"); + let body = read_body_capped(resp, 8192).await; + + assert!( + body.read_failed, + "a body cut off mid-stream must be reported as a failed read, not as a \ + body that ended: got {:?}", + body.text + ); + assert!( + !body.truncated, + "the cap did not cut this one; 5 bytes are nowhere near 8 KiB" + ); + } + + /// The must-not half: a body that ends cleanly must NOT be flagged, or the flag + /// means nothing and every terminal starts claiming the node went quiet. + #[tokio::test] + async fn read_body_capped_does_not_flag_a_clean_body() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/ok") + .with_status(500) + .with_body("node said this") + .create_async() + .await; + let resp = reqwest::get(format!("{}/ok", server.url())).await.unwrap(); + let body = read_body_capped(resp, 8192).await; + + assert_eq!(body.text, "node said this"); + assert!(!body.read_failed, "a complete body is not a failed read"); + assert!(!body.truncated, "a complete body is not a truncated one"); + } + + /// Answer one request with headers promising more body than gets written, then + /// close the connection. Returns the listener's address. + async fn spawn_short_body_listener() -> std::net::SocketAddr { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + let mut scratch = [0u8; 1024]; + let _ = sock.read(&mut scratch).await; + let _ = sock + .write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 64\r\n\r\nshort") + .await; + let _ = sock.flush().await; + // Drop closes the socket with 59 of the promised bytes never sent. + }); + addr + } + + // ── total request timeout ─────────────────────────────────────────── + + /// The client's timeout is a TOTAL request timeout, so it bounds a download and + /// not just the handshake. `gl ipfs get`'s documentation leans on exactly that: + /// the wall-clock deadline covers the search and deliberately stops at the + /// response headers, and this timeout is the only thing left bounding the body. + /// + /// Driven at 250ms through `with_timeout`, the seam `new` itself calls with + /// `TOTAL_REQUEST_TIMEOUT`, because a test at the shipped 30s has no place in this + /// suite. What that costs is the value; what it proves is the SHAPE, which is the + /// part in doubt: that the deadline keeps running once the headers have landed. A + /// timeout that covered only the handshake would let this request hang until the + /// listener gives up. + #[tokio::test] + async fn total_timeout_cuts_off_a_body_that_outruns_it() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + let mut scratch = [0u8; 1024]; + let _ = sock.read(&mut scratch).await; + // Headers land immediately, then the body stalls indefinitely. + let _ = sock + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1024\r\n\r\nfirst") + .await; + let _ = sock.flush().await; + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + }); + + let client = NodeClient::with_timeout( + format!("http://{addr}"), + None, + std::time::Duration::from_millis(250), + ); + let started = std::time::Instant::now(); + let resp = client.get("/slow").await.expect("headers arrive promptly"); + assert_eq!( + resp.status(), + 200, + "the stall is in the body, not the status" + ); + let err = resp + .bytes() + .await + .expect_err("a body still arriving past the total timeout must be cut off"); + let elapsed = started.elapsed(); + + assert!( + err.is_timeout(), + "the body read must end in a timeout, got: {err}" + ); + assert!( + elapsed < std::time::Duration::from_secs(5), + "the timeout must fire on its own schedule, not wait out the listener; \ + took {elapsed:?}" + ); + } + + #[test] + fn shipped_client_uses_the_documented_total_timeout() { + // The scaled-down test above proves the shape at 250ms. This pins the value + // `new` actually ships, so the two together cover the documented behaviour. + assert_eq!(TOTAL_REQUEST_TIMEOUT, std::time::Duration::from_secs(30)); + } + #[test] fn sanitize_strips_controls_bidi_and_caps_length() { // C0 (ESC/BEL) and the Cf bidi override (U+202E) are both removed; the diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index c264997b..2cf0e23a 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -294,7 +294,9 @@ async fn cmd_get_inner( } let retry_after = parse_retry_after(resp.headers()); - let (raw, truncated) = read_body_capped(resp, 8 * 1024).await; + let body = read_body_capped(resp, 8 * 1024).await; + let (raw, truncated) = (body.text, body.truncated); + let read_failed = body.read_failed; let parsed = serde_json::from_str::(&raw).ok(); let code = parsed.as_ref().and_then(|v| v["error"].as_str()); let node_msg = parsed @@ -326,7 +328,7 @@ async fn cmd_get_inner( }; let Some(next) = resume_with else { - let msg = sanitize_node_msg(node_msg); + let msg = node_tail(node_msg, read_failed); if code == Some("search_incomplete") { // Naming the bound, never echoing the value: a rejected token is // node-chosen text and has no business in a terminal message. @@ -419,6 +421,22 @@ fn surface_resume(cid: &str, token: Option<&str>) { } } +/// Render the tail of a terminal message: what the node said, sanitized, plus the +/// fact that its body did not finish arriving when that is what happened. +/// +/// A read that fails mid-body is not the same as a node with nothing to say, and the +/// two used to render identically. A 500 whose body died in transit produced an empty +/// message and the terminal read `node returned 500: `, which reports the node as +/// silent when the truth is that the connection broke before it could be heard. +fn node_tail(node_msg: &str, read_failed: bool) -> String { + let msg = sanitize_node_msg(node_msg); + match (read_failed, msg.is_empty()) { + (false, _) => msg, + (true, true) => "the response body could not be read".to_string(), + (true, false) => format!("{msg} (the response body could not be read in full)"), + } +} + /// `Retry-After` in delta-seconds. Absent, non-numeric, or an HTTP-date all fall /// back to one second; the caller clamps whatever comes back. fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> Duration { @@ -1784,6 +1802,133 @@ mod tests { m.assert_async().await; } + /// The transport-error terminal, which is the one arm of the token surfacing that + /// mockito cannot reach: its server outlives the call, and an unmatched route + /// answers 501, so a request always gets a response. + /// + /// A raw listener is what reproduces it. Rung 1 is a real `search_incomplete` 503 + /// carrying a valid continuation, answered with `Connection: close` so reqwest + /// opens a fresh connection for rung 2. Rung 2 is ACCEPTED and then dropped + /// without a byte written, which is what a reset mid-ladder looks like to the + /// client. The ladder ends holding a token that still points at a real position, + /// and losing it there means the only way forward is a bare re-run that restarts + /// at row 0 and re-spends the caller's per-IP budget. + /// + /// MUTATION (RED): drop the `surface_resume` call in the transport-error arm. + #[tokio::test] + async fn test_cmd_get_transport_failure_mid_ladder_surfaces_the_held_token() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + reset_diag(); + let t = make_token("t1"); + let body = incomplete_body(Some(&t), "scan truncated"); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let connections = Arc::new(AtomicUsize::new(0)); + let seen = connections.clone(); + + tokio::spawn(async move { + // Rung 1: a complete 503 with a continuation, then close the connection so + // rung 2 has to dial again. + let (mut sock, _) = listener.accept().await.unwrap(); + seen.fetch_add(1, Ordering::SeqCst); + let mut scratch = [0u8; 2048]; + let _ = sock.read(&mut scratch).await; + let resp = format!( + "HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n\ + Retry-After: 0\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = sock.write_all(resp.as_bytes()).await; + let _ = sock.flush().await; + drop(sock); + + // Rung 2: accept and hang up without a response. + let (sock, _) = listener.accept().await.unwrap(); + seen.fetch_add(1, Ordering::SeqCst); + drop(sock); + }); + + let err = cmd_get( + "bafkreireset".to_string(), + format!("http://{addr}"), + None, + None, + ) + .await + .expect_err("a connection dropped mid-ladder must be an error"); + let told = told(&err); + + assert_eq!( + connections.load(Ordering::SeqCst), + 2, + "the fixture must actually reach rung 2, or the transport arm was never \ + exercised" + ); + assert!( + told.contains(&t) && told.contains(&format!("--scan {t}")), + "a transport failure ends the ladder still holding a usable token, so it \ + must come back with the invocation that resumes from it, got: {told}" + ); + assert!( + told.contains("bafkreireset"), + "the failure must still name the CID it was fetching, got: {told}" + ); + } + + /// A terminal whose body FAILED to arrive must say so, not report the node as + /// silent. + /// + /// The listener answers 500, promises 512 bytes, writes none, and hangs up. The + /// read comes back empty, and the terminal used to render that as + /// `node returned 500: ` with nothing after the colon, which reads as a node that + /// sent no message at all. MUTATION (RED): render the tail with + /// `sanitize_node_msg` again and the message ends at the colon. + #[tokio::test] + async fn test_cmd_get_reports_a_body_that_could_not_be_read() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + reset_diag(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + let mut scratch = [0u8; 2048]; + let _ = sock.read(&mut scratch).await; + let _ = sock + .write_all( + b"HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\n\ + Content-Length: 512\r\nConnection: close\r\n\r\n", + ) + .await; + let _ = sock.flush().await; + }); + + let err = cmd_get( + "bafkreicutread".to_string(), + format!("http://{addr}"), + None, + None, + ) + .await + .expect_err("a 500 is an error whatever became of its body"); + let told = told(&err); + + assert!( + told.contains("500"), + "the terminal must still name the status, got: {told}" + ); + assert!( + told.to_lowercase().contains("could not be read"), + "a body that failed mid-read must be reported as unread rather than as an \ + empty message, got: {told}" + ); + assert!( + !told.contains("500: \n") && !told.ends_with("500: "), + "the terminal must not trail off after the colon, got: {told}" + ); + } + /// #173 review (F9): a `search_incomplete` body the 8 KiB read cap CUT SHORT must /// be terminal, not retried. /// diff --git a/crates/gl/src/peer.rs b/crates/gl/src/peer.rs index c0f7bca8..aa420f2e 100644 --- a/crates/gl/src/peer.rs +++ b/crates/gl/src/peer.rs @@ -199,7 +199,7 @@ async fn cmd_add(peer_url: String, node: String, dir: Option) -> Result // in the command: bound the read, and defang the message before it reaches // the terminal through the error return. An announce reply is a DID, a URL // and a count, so 8 KiB is well past what the shape needs. - let (raw, _truncated) = read_body_capped(resp, 8 * 1024).await; + let raw = read_body_capped(resp, 8 * 1024).await.text; if let Some(failure) = remote_announce_failure(status, &raw) { anyhow::bail!("{failure}"); @@ -237,7 +237,7 @@ async fn cmd_add(peer_url: String, node: String, dir: Option) -> Result // node (or a MITM on plain http) must not force an unbounded // read. A body that does not parse stays `Null`, which still // routes a non-success status to the warning. - let (raw, _truncated) = read_body_capped(resp, 8 * 1024).await; + let raw = read_body_capped(resp, 8 * 1024).await.text; let result: Value = serde_json::from_str(&raw).unwrap_or(Value::Null); let (is_warning, line) = local_add_report(status, &result); if is_warning { diff --git a/crates/gl/src/sync.rs b/crates/gl/src/sync.rs index 60950d99..c4e830c3 100644 --- a/crates/gl/src/sync.rs +++ b/crates/gl/src/sync.rs @@ -46,7 +46,7 @@ pub async fn run(args: SyncArgs) -> Result<()> { if !status.is_success() { // Bound the read: a hostile or broken node must not force an // unbounded allocation just to surface a denial (INV-6, read half). - let (raw, _truncated) = read_body_capped(resp, 8 * 1024).await; + let raw = read_body_capped(resp, 8 * 1024).await.text; let msg = serde_json::from_str::(&raw) .ok() .and_then(|v| { @@ -261,7 +261,7 @@ mod tests { .create_async() .await; let resp = reqwest::get(format!("{}/big", server.url())).await.unwrap(); - let (out, _truncated) = read_body_capped(resp, 8192).await; + let out = read_body_capped(resp, 8192).await.text; assert!(out.len() <= 8192, "read not bounded: {} bytes", out.len()); assert!(!out.is_empty(), "expected some body"); } From ba44a018ec8945b447977d89fc6a10d98fc1bb7b Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Thu, 13 Aug 2026 17:34:14 -0500 Subject: [PATCH 65/77] fix(core): share the origin-scoped redirect rule with the git helper The previous commit stopped the CLI carrying request signatures to a host the node redirected it to, and left the other signing client alone. That one is the git remote helper, which is what actually runs on clone, fetch and push, and it had no redirect policy at all. A push signs from its first request, so a node answering 302 handed out a working credential, and on a 307 or 308 the pack body went with it. The rule now lives in core and both clients use it, so the two cannot drift apart again. Three guards were not holding what they claimed. Bounding the retry wait by the remaining deadline could be deleted with every test still green, which the timing shows was four real seconds of overshoot past the bound the docs advertise; the assertion was loose enough to pass either way and is now tight enough to fail. The refusing-gateway warning was asserted through a test-only mirror rather than the write the user sees, so deleting the write kept the test green. And both no-hint exceptions, the definitive not-found and the wrapped scan, were only ever reached without a token in hand, which is the one state where they do nothing. A response whose body failed partway through the read still replayed the held token for every remaining attempt, spending the whole ladder on one position and then reporting that the object might lie further on. A body cut by the read cap was already terminal for that reason; a body cut by a broken connection is the same unreadable state and is terminal now too. A successful fetch no longer buffers the whole object before writing a byte of it, so a node cannot make the client hold an arbitrary amount of memory inside a time limit that was never a memory limit. The remaining silent skips in the recovery loops now say what went wrong, including the manifest fetch and the body reads, so a failure there is reported rather than looking like an absent blob. Also: the redirect chain bound is off by one against reqwest's own, has a test, and its comment no longer claims to prevent something the request timeout already prevents. --- Cargo.lock | 1 + Cargo.toml | 3 + README.md | 6 +- crates/git-remote-gitlawb/src/main.rs | 195 +++++++++++++- crates/gitlawb-core/Cargo.toml | 1 + crates/gitlawb-core/src/lib.rs | 1 + crates/gitlawb-core/src/redirect.rs | 160 ++++++++++++ crates/gl/src/clone.rs | 98 +++++-- crates/gl/src/http.rs | 167 +++++------- crates/gl/src/ipfs_cmd.rs | 355 ++++++++++++++++++++++++-- 10 files changed, 854 insertions(+), 133 deletions(-) create mode 100644 crates/gitlawb-core/src/redirect.rs diff --git a/Cargo.lock b/Cargo.lock index b7050bc6..556d857d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3427,6 +3427,7 @@ dependencies = [ "sha2", "thiserror 2.0.18", "tokio", + "url", "uuid", "zeroize", ] diff --git a/Cargo.toml b/Cargo.toml index b2fd6c07..9b8b4684 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,9 @@ chrono = { version = "0.4", features = ["serde"] } uuid = { version = "1", features = ["v4"] } # http client reqwest = { version = "0.12", features = ["blocking", "json", "multipart", "rustls-tls"], default-features = false } +# URL parsing (what reqwest::Url re-exports, so the shared redirect predicate can +# take a parsed URL without pulling reqwest into gitlawb-core) +url = "2" # HMAC hmac = "0.12" diff --git a/README.md b/README.md index de85a271..a3145340 100644 --- a/README.md +++ b/README.md @@ -246,8 +246,10 @@ deadline. They are not unbounded, though. The client's own 30 second HTTP timeou is a total request timeout, running from the start of a request until its body has finished, so a transfer still going 30 seconds after its request began is cut off. Waits between attempts never run past the deadline either, so a single run -tops out around 90 seconds: the deadline plus the 30 second timeout covering the -last attempt. +spends at most around 90 seconds on the network: the deadline plus the 30 second +timeout covering the last attempt. Writing the object out sits outside both +bounds, so piping into a reader that stops reading can hold the command open +longer than that. Two node-side brakes end a ladder early and are reported rather than retried around. A 429 is terminal, because the node's rate-limit window is an hour and diff --git a/crates/git-remote-gitlawb/src/main.rs b/crates/git-remote-gitlawb/src/main.rs index 02e39c3e..ea2ca400 100644 --- a/crates/git-remote-gitlawb/src/main.rs +++ b/crates/git-remote-gitlawb/src/main.rs @@ -199,9 +199,7 @@ fn handle_connect( other => bail!("unsupported git service: {other}"), } - let client = reqwest::blocking::Client::builder() - .timeout(std::time::Duration::from_secs(300)) - .build()?; + let client = build_http_client()?; // ── Phase 1: ref advertisement (GET /info/refs?service=) ───────── // @@ -320,6 +318,56 @@ fn handle_connect( ) } +// ── HTTP client ─────────────────────────────────────────────────────────────── + +/// Total request timeout. A pack transfer can be large and slow, so this is far +/// wider than the CLI's. +const HTTP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300); + +/// The one client both phases use, with the redirect policy the signing surfaces +/// need. +/// +/// Every request this client sends can carry RFC 9421 `Signature` and +/// `Signature-Input` headers: a push signs from the first request, and a fetch of a +/// private repo signs on the retry. reqwest strips only `Authorization`, `Cookie`, +/// `Proxy-Authorization` and `WWW-Authenticate` across hosts, so under the default +/// `Policy::limited(10)` those signature headers rode a 302 to whatever origin the +/// node named, and on a 307/308 the pack body went with them. Scope the follow to +/// the origin that issued the redirect, which is the same predicate `gl` uses. +fn build_http_client() -> Result { + Ok(reqwest::blocking::Client::builder() + .timeout(HTTP_TIMEOUT) + .redirect(reqwest::redirect::Policy::custom(same_origin_redirect)) + .build()?) +} + +/// Refuse any redirect that leaves the issuing origin, and bound the chain. +/// +/// Refusal is `stop`, not `error`: the 3xx comes back as an ordinary response and +/// the caller reports it through the status path it already has. +/// +/// `Policy::custom` replaces reqwest's built-in limit, so the chain bound is +/// restated. It is not what makes the request finite (`HTTP_TIMEOUT` covers the whole +/// chain); it is what keeps a node that redirects to itself from costing a request +/// per round trip until that timeout. `>` and not `>=` because reqwest pushes the +/// redirecting URL onto `previous` before consulting the policy, which is how +/// `Policy::limited` reads the same counter. +fn same_origin_redirect(attempt: reqwest::redirect::Attempt<'_>) -> reqwest::redirect::Action { + let Some(previous) = attempt.previous().last() else { + // Unreachable through reqwest, which pushes the redirecting URL first, but + // the safe reading of "cannot prove same-origin" is to refuse. + return attempt.stop(); + }; + if attempt.previous().len() > gitlawb_core::redirect::MAX_REDIRECTS { + return attempt.stop(); + } + if gitlawb_core::redirect::may_follow(previous, attempt.url()) { + attempt.follow() + } else { + attempt.stop() + } +} + // ── Smart-protocol request builders ─────────────────────────────────────────── const USER_AGENT: &str = "git/2.0 git-remote-gitlawb/0.1.0"; @@ -858,6 +906,147 @@ mod tests { String::from_utf8_lossy(&buf).into_owned() } + /// The signed headers must not survive a redirect off the node's origin, on + /// EITHER phase. + /// + /// This is the binary git runs for `clone`, `fetch` and `push`. It built its + /// client with a timeout and nothing else, so it ran reqwest's default + /// `Policy::limited(10)`, and reqwest's cross-host header stripping covers + /// `Authorization`, `Cookie`, `Proxy-Authorization` and `WWW-Authenticate` only. + /// `Signature` and `Signature-Input` came straight through. A push signs from the + /// first request, so a hostile node answering 302 was handed a working credential, + /// and a 307/308 would have taken the pack body along. + /// + /// Two mockito servers are two ports on one host, which is the boundary this + /// policy draws. The second server answers everything and expects nothing, with a + /// second mock matching on the `signature` header also at zero, so a followed + /// redirect fails whether or not the signature came with it. Phase 1 is driven + /// with a 302 and Phase 2 with a 308, the status that would carry the body. + /// + /// MUTATION (RED): drop the `.redirect(...)` line from `build_http_client`. + #[test] + fn signed_requests_do_not_follow_a_redirect_off_the_node_origin() { + let kp = Keypair::generate(); + let client = build_http_client().unwrap(); + + let mut elsewhere = mockito::Server::new(); + let never = elsewhere + .mock("GET", mockito::Matcher::Any) + .with_status(200) + .with_body("bytes from the redirect target") + .expect(0) + .create(); + let never_post = elsewhere + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .expect(0) + .create(); + let signature_seen_get = elsewhere + .mock("GET", mockito::Matcher::Any) + .match_header("signature", mockito::Matcher::Any) + .with_status(200) + .expect(0) + .create(); + let signature_seen_post = elsewhere + .mock("POST", mockito::Matcher::Any) + .match_header("signature", mockito::Matcher::Any) + .with_status(200) + .expect(0) + .create(); + + let mut node = mockito::Server::new(); + let repo_base = format!("{}/zOwner/myrepo", node.url()); + let bounce_get = node + .mock("GET", mockito::Matcher::Regex(r"/info/refs".to_string())) + .with_status(302) + .with_header("location", &format!("{}/info/refs", elsewhere.url())) + .expect(1) + .create(); + let bounce_post = node + .mock( + "POST", + mockito::Matcher::Regex(r"/git-receive-pack$".to_string()), + ) + .with_status(308) + .with_header("location", &format!("{}/git-receive-pack", elsewhere.url())) + .expect(1) + .create(); + + let refs_url = format!("{repo_base}/info/refs?service=git-receive-pack"); + let advertisement = build_advertisement_request(&client, &refs_url, Some(&kp)) + .send() + .unwrap(); + + let body = b"0009done\n".to_vec(); + let post_url = format!("{repo_base}/git-receive-pack"); + let pack_post = + build_pack_post_request(&client, &post_url, "git-receive-pack", &body, Some(&kp)) + .body(body.clone()) + .send() + .unwrap(); + + // The other origin first: it is the assertion that names the leak, and it must + // be the one that speaks when a followed redirect makes every one of these + // fail at once. + never.assert(); + never_post.assert(); + signature_seen_get.assert(); + signature_seen_post.assert(); + bounce_get.assert(); + bounce_post.assert(); + + assert_eq!( + advertisement.status(), + 302, + "a refused redirect stops rather than errors, so the caller sees the 3xx" + ); + assert!( + !advertisement + .text() + .unwrap() + .contains("bytes from the redirect target"), + "the redirect target's bytes must never reach the caller" + ); + assert_eq!( + pack_post.status(), + 308, + "the pack POST stops at the redirect too" + ); + } + + /// The other direction: a same-origin redirect is still followed, so a node + /// fronted by a proxy that normalizes a path keeps working. Without this the + /// policy could be tightened to `Policy::none()` and nothing would notice. + #[test] + fn a_same_origin_redirect_is_still_followed() { + let kp = Keypair::generate(); + let client = build_http_client().unwrap(); + + let mut node = mockito::Server::new(); + let bounce = node + .mock("GET", "/zOwner/myrepo/info/refs") + .with_status(301) + .with_header("location", "/zOwner/myrepo/info/refs/") + .expect(1) + .create(); + let target = node + .mock("GET", "/zOwner/myrepo/info/refs/") + .with_status(200) + .with_body("normalized") + .expect(1) + .create(); + + let refs_url = format!("{}/zOwner/myrepo/info/refs", node.url()); + let resp = build_advertisement_request(&client, &refs_url, Some(&kp)) + .send() + .unwrap(); + + assert_eq!(resp.status(), 200); + assert_eq!(resp.text().unwrap(), "normalized"); + bounce.assert(); + target.assert(); + } + /// The regression that round-1 missed: the Phase-2 `git-upload-pack` POST was /// left unsigned, so an owner's fetch of a private repo cleared the (now signed) /// advertisement and then 404'd on the pack POST. Drive BOTH request builders diff --git a/crates/gitlawb-core/Cargo.toml b/crates/gitlawb-core/Cargo.toml index 3ba05f1c..d2b3c05b 100644 --- a/crates/gitlawb-core/Cargo.toml +++ b/crates/gitlawb-core/Cargo.toml @@ -22,6 +22,7 @@ multihash-codetable = { workspace = true } cid = { workspace = true } chrono = { workspace = true } uuid = { workspace = true } +url = { workspace = true } zeroize = { version = "1", features = ["derive"] } pkcs8 = { version = "0.10", features = ["pem", "std"] } curve25519-dalek = "4" diff --git a/crates/gitlawb-core/src/lib.rs b/crates/gitlawb-core/src/lib.rs index bda927be..ae6564a9 100644 --- a/crates/gitlawb-core/src/lib.rs +++ b/crates/gitlawb-core/src/lib.rs @@ -5,6 +5,7 @@ pub mod encrypt; pub mod error; pub mod http_sig; pub mod identity; +pub mod redirect; pub mod sanitize; pub mod scan_token; pub mod ucan; diff --git a/crates/gitlawb-core/src/redirect.rs b/crates/gitlawb-core/src/redirect.rs new file mode 100644 index 00000000..bfb5738c --- /dev/null +++ b/crates/gitlawb-core/src/redirect.rs @@ -0,0 +1,160 @@ +//! Redirect policy shared by every gitlawb HTTP client that signs its requests. +//! +//! Both `gl` (async) and `git-remote-gitlawb` (blocking) attach RFC 9421 +//! `Signature` and `Signature-Input` headers, and reqwest strips only +//! `Authorization`, `Cookie`, `Proxy-Authorization` and `WWW-Authenticate` when a +//! redirect crosses hosts. A signature would survive that hop, and it binds +//! `@method`, `@path` and `content-digest` with no authority component, so a node +//! answering 302 could hand a working credential to a host of its choosing and read +//! as the caller anywhere until the clock-skew window closes. On a 307/308 the +//! request body goes along with it, which for the remote helper is the pack. +//! +//! The decision lives here rather than in either client because the two used to +//! disagree: `gl` was scoped to the origin while the remote helper, the binary that +//! actually runs `git clone gitlawb://`, still ran reqwest's default and followed +//! anywhere. One predicate is what keeps a future third client from repeating that. +//! +//! The type is `url::Url`, which is what `reqwest::Url` re-exports, so both clients +//! pass their attempt URLs straight in. + +/// Longest redirect chain followed. `reqwest::redirect::Policy::custom` replaces +/// reqwest's built-in limit, so the bound has to be restated by every client that +/// installs a custom policy; the value is reqwest's own default. +/// +/// This counts FOLLOWS, matching `Policy::limited`: reqwest pushes the redirecting +/// URL onto `previous` before consulting the policy, and `Limit(max)` refuses once +/// `previous.len() > max`, so a caller comparing against this constant must use `>` +/// too or it permits one hop fewer than it says. +pub const MAX_REDIRECTS: usize = 10; + +/// Follow a redirect only when it stays on the origin that issued it. +/// +/// `Policy::none()` would have been the simpler answer, but same-origin redirects +/// are legitimate here (a node fronted by a proxy that upgrades http to https, or +/// normalizes a trailing slash), so the policy is scoped to the origin rather than +/// switched off. +/// +/// Host and port must match exactly. Port is compared as `Url::port`, which is +/// `None` for a scheme's default port, so http -> https on the same host compares +/// equal while http -> http on a different port does not. A downgrade from https to +/// http is refused as well: the target is the same host, but the signature would go +/// out in cleartext, which is the same credential leak by a slower route. +/// +/// Host comparison rides on `url`'s parse-time normalization (lowercasing and IDN +/// -> punycode), so the spellings an attacker reaches for do not open a gap. That +/// is a property of the parsed `Url`, not of this function, which is why the test +/// matrix pins it: a move to raw string comparison would silently lose it. +pub fn may_follow(previous: &url::Url, next: &url::Url) -> bool { + let same_origin = next.host_str() == previous.host_str() && next.port() == previous.port(); + let downgraded = previous.scheme() == "https" && next.scheme() != "https"; + same_origin && !downgraded +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every branch of the decision, both ways, plus the spellings that would slip + /// past a comparison less careful than `Url`'s own normalization. + #[test] + fn may_follow_covers_each_origin_branch() { + let url = |s: &str| url::Url::parse(s).unwrap(); + let cases: &[(&str, &str, bool, &str)] = &[ + ( + "http://node.example/a", + "http://node.example/b", + true, + "same origin, different path", + ), + ( + "http://node.example/a", + "https://node.example/a", + true, + "http to https on one host: both ports are the scheme default", + ), + ( + "https://node.example/a", + "https://node.example/a/", + true, + "trailing-slash normalization", + ), + ( + "https://node.example:8443/a", + "https://node.example:8443/b", + true, + "same explicit port", + ), + ( + "http://node.example/a", + "http://attacker.example/a", + false, + "different host", + ), + ( + "http://node.example/a", + "http://node.example:8080/a", + false, + "same host, different port", + ), + ( + "https://node.example/a", + "http://node.example/a", + false, + "https downgraded to cleartext on the same host", + ), + ( + "https://node.example/a", + "http://node.example:443/a", + false, + "a downgrade dressed up as the https port", + ), + // The rows below pass today because `Url::parse` normalizes the host, not + // because anything here compares case-insensitively or decodes IDN. They + // are the variants an attacker reaches for, so they are pinned: swapping + // this predicate for a raw string comparison must break the suite. + ( + "https://node.example/a", + "https://NODE.EXAMPLE/b", + true, + "same host in a different case: parse lowercases it", + ), + ( + "https://node.example/a", + "https://node.example./b", + false, + "a trailing dot is a different host to url, so the redirect is refused", + ), + ( + "https://exämple.test/a", + "https://xn--exmple-cua.test/b", + true, + "unicode host and its punycode spelling are one host after parse", + ), + ( + "https://node.example/a", + "https://user:pw@node.example/b", + true, + "userinfo is not part of the origin: same host, still followed", + ), + ( + "https://node.example/a", + "https://node.example@attacker.example/b", + false, + "the node's name smuggled into userinfo: the host is the attacker's", + ), + ( + "https://node.example/a", + "https://attacker.example#node.example/b", + false, + "the node's name pushed into the fragment: the host is the attacker's", + ), + ]; + for (previous, next, expected, why) in cases { + assert_eq!( + may_follow(&url(previous), &url(next)), + *expected, + "{previous} -> {next} ({why})" + ); + } + } +} diff --git a/crates/gl/src/clone.rs b/crates/gl/src/clone.rs index fc90316d..926a2d57 100644 --- a/crates/gl/src/clone.rs +++ b/crates/gl/src/clone.rs @@ -27,12 +27,41 @@ use crate::identity::load_keypair_from_dir; /// can assert the loop actually says what it skipped rather than only that it /// returned nothing. fn warn_skip(oid: &str, why: &str) { - let line = sanitize_node_msg(&format!( + emit_warning(&format!( "warning: could not fetch encrypted blob {oid}: {why}; skipping" )); - eprintln!("{line}"); - #[cfg(test)] - tests::record_warn(&line); +} + +/// The same report for a manifest rather than a blob. The manifest is one step +/// earlier in the same recovery: losing it silently means every blob it would have +/// named is missing with no reason given anywhere. +fn warn_skip_manifest(id: &str, why: &str) { + emit_warning(&format!( + "warning: could not fetch blob manifest {id}: {why}; skipping" + )); +} + +/// Sanitize a warning and write it, once. +/// +/// The write goes to [`warn_sink`], which is stderr in a normal build and the tests' +/// per-thread mirror under `cfg(test)`. That indirection is the point: the mirror +/// used to sit BESIDE an `eprintln!`, so deleting the user-visible write left every +/// assertion on the mirror green and the shipped behaviour could be removed with the +/// tests unchanged. There is one write now, and the tests observe it. +fn emit_warning(line: &str) { + use std::io::Write; + let line = sanitize_node_msg(line); + let _ = writeln!(warn_sink(), "{line}"); +} + +#[cfg(not(test))] +fn warn_sink() -> impl std::io::Write { + std::io::stderr() +} + +#[cfg(test)] +fn warn_sink() -> impl std::io::Write { + tests::WarnMirror } #[derive(Args)] @@ -332,10 +361,25 @@ async fn recover_encrypted_blobs( .await { Ok(r) if r.status().is_success() => r, - _ => continue, + // The node path has the same silent exit the gateway path had: a 403, a + // 404, or a dead connection all reached the caller as "blob not + // recoverable" with no reason attached. One unreachable blob still must + // not end the recovery of the rest, so it warns and moves on. + Ok(r) => { + warn_skip(oid, &format!("node returned {}", r.status())); + continue; + } + Err(e) => { + warn_skip(oid, &format!("node request failed: {e}")); + continue; + } }; - let Ok(envelope) = env_resp.bytes().await else { - continue; + let envelope = match env_resp.bytes().await { + Ok(b) => b, + Err(e) => { + warn_skip(oid, &format!("reading the envelope failed: {e}")); + continue; + } }; let plaintext = match open_blob(&envelope, keypair) { Ok(p) => p, @@ -662,7 +706,14 @@ async fn recover_from_arweave( for r in refs { let m = match client.get(format!("{ag}/{}", r.id)).send().await { Ok(resp) if resp.status().is_success() => resp, - _ => continue, + Ok(resp) => { + warn_skip_manifest(&r.id, &format!("gateway returned {}", resp.status())); + continue; + } + Err(e) => { + warn_skip_manifest(&r.id, &format!("gateway request failed: {e}")); + continue; + } }; if let Ok(parsed) = m.json::().await { manifests.push((parsed, r.height)); @@ -720,8 +771,15 @@ async fn recover_from_arweave( continue; } }; - let Ok(envelope) = env_resp.bytes().await else { - continue; + // A body that dies part-way through is the same mid-read failure the capped + // read exists to stop rendering as silence, and it sits INSIDE the loop whose + // status arms were already fixed. + let envelope = match env_resp.bytes().await { + Ok(b) => b, + Err(e) => { + warn_skip(&oid, &format!("reading the envelope failed: {e}")); + continue; + } }; // open_blob succeeds only if this caller is a recipient: this is the // authorization gate (no node, no DID check needed). @@ -846,11 +904,21 @@ mod tests { static WARNINGS: RefCell = const { RefCell::new(String::new()) }; } - pub(super) fn record_warn(msg: &str) { - WARNINGS.with(|w| { - w.borrow_mut().push_str(msg); - w.borrow_mut().push('\n'); - }); + /// The `cfg(test)` warning sink. `emit_warning` writes here instead of to + /// stderr, so the assertions below observe the shipped write rather than a copy + /// made beside it: delete that write and they go red. + pub(super) struct WarnMirror; + + impl std::io::Write for WarnMirror { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let text = String::from_utf8_lossy(buf).into_owned(); + WARNINGS.with(|w| w.borrow_mut().push_str(&text)); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } } fn reset_warnings() { diff --git a/crates/gl/src/http.rs b/crates/gl/src/http.rs index c1f610a7..9a4614f9 100644 --- a/crates/gl/src/http.rs +++ b/crates/gl/src/http.rs @@ -19,33 +19,26 @@ const MAX_ICAPTCHA_RETRIES: usize = 2; /// response body, so it bounds a slow download and not just a slow handshake. const TOTAL_REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); -/// Longest redirect chain followed. `Policy::custom` replaces reqwest's built-in -/// limit, so the loop bound has to be restated here; the value is reqwest's own -/// default. Same-origin redirects can cycle, and without this a node answering 302 -/// to itself would spin forever. -const MAX_REDIRECTS: usize = 10; - -/// Follow a redirect only when it stays on the origin that issued it. +/// Follow a redirect only when it stays on the origin that issued it, and only for +/// as long as the chain bound allows. /// -/// Every request this client sends may carry RFC 9421 `Signature` and -/// `Signature-Input` headers, and reqwest strips only `Authorization`, `Cookie`, -/// `Proxy-Authorization` and `WWW-Authenticate` when a redirect crosses hosts. The -/// signature would survive, and it binds `@method`, `@path` and `content-digest` -/// with no authority component, so a node answering 302 could hand a working -/// credential to a host of its choosing and read as the caller anywhere for as long -/// as the node's clock-skew window lasts. +/// The origin decision itself is [`gitlawb_core::redirect::may_follow`], shared with +/// `git-remote-gitlawb` so the two signing clients cannot drift apart on it. Refusal +/// is `stop`, not `error`: the 3xx comes back as an ordinary response and each caller +/// reports it through the status path it already has. /// -/// `Policy::none()` would have been the simpler answer, but same-origin redirects -/// are legitimate here (a node fronted by a proxy that upgrades http to https, or -/// normalizes a trailing slash), so the policy is scoped to the origin rather than -/// switched off. Refusal is `stop`, not `error`: the 3xx comes back as an ordinary -/// response and each caller reports it through the status path it already has. +/// `Policy::custom` replaces reqwest's built-in limit, so the chain bound is restated +/// here. Same-origin redirects can cycle, and this is what stops a node answering 302 +/// to itself from being followed indefinitely. It is not what makes the request +/// finite: `.timeout(...)` on the same builder is a TOTAL request timeout covering the +/// whole chain, so without this bound the worst case is a 30 second spin, not an +/// endless one. The bound is what keeps that spin from costing the node a request per +/// round trip for the full 30 seconds. /// -/// Host and port must match exactly. Port is compared as `Url::port`, which is -/// `None` for a scheme's default port, so http -> https on the same host compares -/// equal while http -> http on a different port does not. A downgrade from https to -/// http is refused as well: the target is the same host, but the signature would go -/// out in cleartext, which is the same credential leak by a slower route. +/// `>` and not `>=`: reqwest pushes the redirecting URL onto `previous` before +/// consulting the policy, so on the first redirect `previous.len()` is already 1, and +/// `>=` would permit `MAX_REDIRECTS - 1` follows. `Policy::limited(max)` refuses at +/// `previous.len() > max`, and matching it is the point of reusing its value. fn same_origin_redirect(attempt: reqwest::redirect::Attempt<'_>) -> reqwest::redirect::Action { let Some(previous) = attempt.previous().last() else { // No previous URL to compare against. Unreachable through reqwest, which @@ -53,25 +46,16 @@ fn same_origin_redirect(attempt: reqwest::redirect::Attempt<'_>) -> reqwest::red // reading of "cannot prove same-origin" is to refuse. return attempt.stop(); }; - if attempt.previous().len() >= MAX_REDIRECTS { + if attempt.previous().len() > gitlawb_core::redirect::MAX_REDIRECTS { return attempt.stop(); } - if may_follow(previous, attempt.url()) { + if gitlawb_core::redirect::may_follow(previous, attempt.url()) { attempt.follow() } else { attempt.stop() } } -/// The decision itself, split out because `redirect::Attempt` cannot be built outside -/// reqwest, so this is the only way to run the scheme and port branches both ways -/// rather than reasoning about them. -fn may_follow(previous: &reqwest::Url, next: &reqwest::Url) -> bool { - let same_origin = next.host_str() == previous.host_str() && next.port() == previous.port(); - let downgraded = previous.scheme() == "https" && next.scheme() != "https"; - same_origin && !downgraded -} - pub struct NodeClient { inner: reqwest::Client, pub node_url: String, @@ -792,69 +776,56 @@ mod tests { signature_seen.assert_async().await; } - /// Every branch of the decision, both ways. The end-to-end test above drives one - /// pair of http origins, which is all mockito can serve; the scheme cases and the - /// default-port equivalence have no other way to be run. - #[test] - fn may_follow_covers_each_origin_branch() { - let url = |s: &str| reqwest::Url::parse(s).unwrap(); - let cases: &[(&str, &str, bool, &str)] = &[ - ( - "http://node.example/a", - "http://node.example/b", - true, - "same origin, different path", - ), - ( - "http://node.example/a", - "https://node.example/a", - true, - "http to https on one host: both ports are the scheme default", - ), - ( - "https://node.example/a", - "https://node.example/a/", - true, - "trailing-slash normalization", - ), - ( - "https://node.example:8443/a", - "https://node.example:8443/b", - true, - "same explicit port", - ), - ( - "http://node.example/a", - "http://attacker.example/a", - false, - "different host", - ), - ( - "http://node.example/a", - "http://node.example:8080/a", - false, - "same host, different port", - ), - ( - "https://node.example/a", - "http://node.example/a", - false, - "https downgraded to cleartext on the same host", - ), - ( - "https://node.example/a", - "http://node.example:443/a", - false, - "a downgrade dressed up as the https port", - ), - ]; - for (previous, next, expected, why) in cases { - assert_eq!( - may_follow(&url(previous), &url(next)), - *expected, - "{previous} -> {next} ({why})" - ); - } + /// A node redirecting to itself is same-origin, so the origin predicate follows it + /// every time and only the chain bound ends the loop. Deleting the bound left the + /// whole suite green, because nothing here had ever built a cycle. + /// + /// The route answers 301 pointing back at itself. Bounded, the handler is hit + /// once for the original request plus `MAX_REDIRECTS` follows and the call returns + /// the 301 (a refused redirect stops rather than errors). Unbounded, it runs until + /// the client's total request timeout cuts it off, which is a 30 second spin at the + /// shipped value and a request per round trip for the node. + /// + /// MUTATION (RED): delete the `previous().len()` check. + #[tokio::test] + async fn a_self_redirect_stops_at_the_chain_bound() { + let mut node = Server::new_async().await; + let hits = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let h = hits.clone(); + let loop_route = node + .mock("GET", "/api/v1/loop") + .with_status(301) + .with_header("location", "/api/v1/loop") + .with_body_from_request(move |_req| { + h.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Vec::new() + }) + .expect_at_least(1) + .create_async() + .await; + + let client = NodeClient::with_timeout( + node.url(), + Some(test_keypair()), + std::time::Duration::from_secs(5), + ); + let resp = client + .get_signed("/api/v1/loop") + .await + .expect("the bound must end the chain, not the timeout"); + + assert_eq!( + resp.status(), + 301, + "the chain ends by refusing the next hop, so the last 3xx is what comes back" + ); + assert_eq!( + hits.load(std::sync::atomic::Ordering::SeqCst), + gitlawb_core::redirect::MAX_REDIRECTS + 1, + "one original request plus MAX_REDIRECTS follows, matching what \ + Policy::limited(MAX_REDIRECTS) would have permitted" + ); + loop_route.assert_async().await; } /// The other direction: a same-origin redirect is still followed, so a node diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index 2cf0e23a..3b160a18 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -48,8 +48,10 @@ pub enum IpfsCmd { /// request starts connecting until its body has finished, so a transfer still /// going 30 seconds after its own request began is cut off. Waits between /// attempts are bounded by the time left on the deadline as well as by the 5 - /// second clamp, so the longest a single run can take is about 90 seconds: the - /// deadline, plus the 30 second timeout covering the last attempt. + /// second clamp, so the longest a run can spend on the network is about 90 + /// seconds: the deadline, plus the 30 second timeout covering the last attempt. + /// Writing the object out is not covered by either bound, so piping into a reader + /// that stops reading can hold the command open past that. /// /// A 429 ends the ladder immediately: the node's rate-limit window is an hour, /// so the wait it asks for cannot be honored inside one invocation. A transient @@ -234,8 +236,10 @@ async fn cmd_get_inner( // 30s and not under the deadline: `write_object`'s success read, which ends the run, // and `read_body_capped`'s error read, which on a retryable arm is followed by one // wait. That wait adds no term of its own, because it is bounded by the time LEFT on - // the deadline as well as by the clamp. So the worst case is deadline + 30s, about - // 90s at the shipped defaults. + // the deadline as well as by the clamp. So the worst case ON THE NETWORK is + // deadline + 30s, about 90s at the shipped defaults. `write_object`'s writes to + // stdout are blocking and under neither bound, so a stalled consumer on the other + // end of the pipe can outlast that; nothing here can bound a caller's own reader. let start = tokio::time::Instant::now(); let mut requests = 0usize; loop { @@ -316,12 +320,17 @@ async fn cmd_get_inner( // it already holds. With no token there is nothing to resume, which falls to // the default arm below. // - // A body the cap CUT SHORT is excluded from this arm. A cut body cannot - // parse, so its code reads as absent and an oversized `search_incomplete` + // A body that did not arrive whole is excluded from this arm, whether the + // cap CUT it short or the read FAILED part-way. Either way it cannot parse, + // so its code reads as absent and an oversized or broken `search_incomplete` // would land here and be retried on the OLD token, replaying one position // for every rung while the fresh continuation it offered goes unread. // Unclassifiable is terminal, like any unrecognized code. - Some(_) | None if status == reqwest::StatusCode::SERVICE_UNAVAILABLE && !truncated => { + Some(_) | None + if status == reqwest::StatusCode::SERVICE_UNAVAILABLE + && !truncated + && !read_failed => + { token.clone() } _ => None, @@ -370,6 +379,21 @@ async fn cmd_get_inner( /// Write a successful response: diagnostics to stderr, raw bytes to stdout so the /// output stays pipeable. async fn write_object(resp: reqwest::Response) -> Result<()> { + write_object_to(resp, &mut std::io::stdout()).await +} + +/// `write_object` with the sink as a parameter, so a test can read back what a +/// caller would have received on stdout. `write_object` supplies the real one. +/// +/// The body is STREAMED. `resp.bytes()` buffers the whole object first, so a node +/// answering 200 with a very large body delivered fast made the client allocate all +/// of it before a byte reached stdout; the 30 second client timeout bounds how long +/// that takes, not how much it costs. Chunk-at-a-time the peak is one chunk, and the +/// sibling error read is already capped at 8 KiB. +async fn write_object_to( + mut resp: reqwest::Response, + out: &mut W, +) -> Result<()> { let headers = resp.headers().clone(); if let Some(git_hash) = headers.get("x-git-hash") { diag(&format!( @@ -384,15 +408,13 @@ async fn write_object(resp: reqwest::Response) -> Result<()> { )); } - let bytes = resp.bytes().await.context("failed to read response body")?; - use std::io::Write; + while let Some(chunk) = resp.chunk().await.context("failed to read response body")? { + out.write_all(&chunk).context("failed to write to stdout")?; + } // Flush explicitly rather than leaving the tail to the process-exit flush, which // discards its error: `gl ipfs get > object.bin` onto a full disk or a // closed pipe would otherwise leave a TRUNCATED file behind exit status 0, and on // a content-addressed fetch a silently short object is the worst possible answer. - let stdout = std::io::stdout(); - let mut out = stdout.lock(); - out.write_all(&bytes).context("failed to write to stdout")?; out.flush().context("failed to flush stdout")?; Ok(()) @@ -1788,10 +1810,18 @@ mod tests { hostile Retry-After must not swallow the run: made {calls} calls", MAX_RETRY_AFTER.as_secs() ); + // Tight enough to bind. The waits are also clamped by the time LEFT on the + // deadline, and at 12s that term was free: dropping `.min(left)` let the run + // overshoot to 10.04s and still pass, so the doc claim that waits never run + // past the deadline rested on a term no test could see. With a 6s deadline + // and a 5s clamp the bounded run lands near 6s and the unbounded one near + // 10s, and 8s separates them. assert!( - elapsed < Duration::from_secs(12), - "every wait is bounded by the clamp and by the time left on the 6s \ - deadline, so the run ends near the deadline; took {elapsed:?}" + elapsed < Duration::from_secs(8), + "a wait is bounded by the time LEFT on the 6s deadline as well as by the \ + {}s clamp, so the run ends near the deadline rather than a full clamp \ + past it; took {elapsed:?}", + MAX_RETRY_AFTER.as_secs() ); assert!( told(&err).to_lowercase().contains("deadline"), @@ -1997,4 +2027,299 @@ mod tests { m1.assert_async().await; m2.assert_async().await; } + + /// The other half of the same defect: a `search_incomplete` 503 whose body read + /// FAILED part-way is just as unparseable as one the cap cut short, and it used to + /// fall through to the generic overload arm and be retried on the token ALREADY + /// HELD. Measured before the fix: rung 1 hands back t1, every later rung answers + /// headers plus a cut body, and the ladder made 9 calls with calls 2 through 9 all + /// carrying the identical `?scan=t1`, ending at the cap. That is the replay the + /// truncation exclusion exists to prevent, reached by the other door. + /// + /// mockito cannot express it: it always finishes the response it advertises. The + /// listener promises 512 bytes, writes a handful, and hangs up. + /// + /// MUTATION (RED): drop `&& !read_failed` from the retry arm and the count is 9. + #[tokio::test] + async fn test_cmd_get_unreadable_incomplete_body_is_terminal_not_a_replay() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + reset_diag(); + let t = make_token("t1"); + let complete = incomplete_body(Some(&t), "scan truncated"); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let seen = calls.clone(); + let scans = Arc::new(Mutex::new(Vec::::new())); + let recorded = scans.clone(); + + tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = listener.accept().await else { + return; + }; + let n = seen.fetch_add(1, Ordering::SeqCst); + let mut scratch = [0u8; 4096]; + let read = sock.read(&mut scratch).await.unwrap_or(0); + let request = String::from_utf8_lossy(&scratch[..read]).into_owned(); + if let Some(line) = request.lines().next() { + if let Some(target) = line.split_whitespace().nth(1) { + recorded + .lock() + .unwrap() + .push(scan_of(target).unwrap_or_default()); + } + } + let resp = if n == 0 { + // Rung 1: a complete 503 offering a continuation. + format!( + "HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n\ + Retry-After: 0\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{complete}", + complete.len() + ) + } else { + // Every later rung: headers, then a body that stops part-way. + "HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n\ + Retry-After: 0\r\nContent-Length: 512\r\nConnection: close\r\n\r\n\ + {\"error\":\"search_inc" + .to_string() + }; + let _ = sock.write_all(resp.as_bytes()).await; + let _ = sock.flush().await; + drop(sock); + } + }); + + let err = cmd_get_inner( + "bafkreiunreadable".to_string(), + format!("http://{addr}"), + None, + None, + SCAN_DEADLINE, + MAX_SCAN_RESUMES, + ) + .await + .expect_err("an unclassifiable 503 body must be an error"); + let told = told(&err); + + assert_eq!( + calls.load(Ordering::SeqCst), + 2, + "a body whose read failed must end the ladder, not replay the held token \ + for every remaining rung; the scans seen were {:?}", + scans.lock().unwrap() + ); + assert_eq!( + scans.lock().unwrap().as_slice(), + [String::new(), t.clone()], + "rung 1 carries no token and rung 2 carries the one it was handed" + ); + assert!( + told.contains(&t) && told.contains(&format!("--scan {t}")), + "the still-held continuation and its resuming invocation must be \ + surfaced, got: {told}" + ); + } + + /// A 404 after a resume is an ANSWER, so it must not come with a resume hint that + /// contradicts it. Deleting the `status != NOT_FOUND` guard (replacing it with + /// `if true`) left the suite green, because no test had ever reached that arm + /// holding a token, which is the only state in which the guard does anything. + /// + /// Rung 1 hands back a valid continuation, rung 2 answers 404. + /// + /// MUTATION (RED): replace the guard with `if true`. + #[tokio::test] + async fn test_cmd_get_a_404_after_a_resume_offers_no_hint() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + + let rung1 = server + .mock("GET", "/ipfs/bafkreignotfound") + .match_query(mockito::Matcher::Missing) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + let rung2 = server + .mock("GET", "/ipfs/bafkreignotfound") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(404) + .with_header("content-type", "application/json") + .with_body(r#"{"error":"not_found","message":"no such object"}"#) + .expect(1) + .create_async() + .await; + + let err = cmd_get("bafkreignotfound".to_string(), server.url(), None, None) + .await + .expect_err("a 404 is still an error exit"); + let told = told(&err); + + assert!( + told.contains("404"), + "the terminal must name the status, got: {told}" + ); + assert!( + !told.contains("--scan") && !told.contains(&t), + "a definitive 404 is an answer; a resume hint beside it would invite a \ + re-run that cannot do better, got: {told}" + ); + + rung1.assert_async().await; + rung2.assert_async().await; + } + + /// `search_incomplete` with NO continuation is the node's "the scan wrapped and + /// finished" signal, so that arm deliberately withholds the hint too. Adding a + /// `surface_resume` call to it left the suite green for the same reason: nothing + /// reached it holding a token. + /// + /// MUTATION (RED): add `surface_resume(&cid, token.as_deref());` to the + /// no-continuation branch. + #[tokio::test] + async fn test_cmd_get_a_wrapped_scan_after_a_resume_offers_no_hint() { + reset_diag(); + let mut server = mockito::Server::new_async().await; + let t = make_token("t1"); + + let rung1 = server + .mock("GET", "/ipfs/bafkreigwrapped") + .match_query(mockito::Matcher::Missing) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(Some(&t), "scan truncated")) + .expect(1) + .create_async() + .await; + let rung2 = server + .mock("GET", "/ipfs/bafkreigwrapped") + .match_query(mockito::Matcher::Exact(format!("scan={t}"))) + .with_status(503) + .with_header("content-type", "application/json") + .with_header("retry-after", "0") + .with_body(incomplete_body(None, "the scan wrapped")) + .expect(1) + .create_async() + .await; + + let err = cmd_get("bafkreigwrapped".to_string(), server.url(), None, None) + .await + .expect_err("an incomplete scan with nothing to resume from is an error"); + let told = told(&err); + + assert!( + told.contains("offered no continuation token"), + "the terminal must say why it stopped, got: {told}" + ); + assert!( + !told.contains("--scan") && !told.contains(&t), + "a wrapped scan has nowhere further to go, so a resume hint here would \ + invite a re-run that cannot find more, got: {told}" + ); + + rung1.assert_async().await; + rung2.assert_async().await; + } + + /// The success path streams. `resp.bytes()` buffered the whole object first, so a + /// hostile node answering 200 with a very large body delivered fast made the + /// client allocate all of it before a byte reached stdout, while the sibling error + /// read was capped at 8 KiB. + /// + /// What is asserted here is the CORRECTNESS of streaming, not the allocation: a + /// body far larger than one chunk must arrive at the sink whole, in order, byte + /// for byte. A chunk loop that dropped or reordered a chunk would be the obvious + /// way to get the memory right and the object wrong, and on a content-addressed + /// fetch that is the worse failure. + #[tokio::test] + async fn write_object_streams_a_large_body_through_intact() { + reset_diag(); + // 4 MiB of a non-repeating pattern, well past any single chunk. + let payload: Vec = (0..4 * 1024 * 1024u32).map(|i| (i % 251) as u8).collect(); + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("GET", "/ipfs/bafkreibig") + .with_status(200) + .with_header("x-git-hash", "deadbeef") + .with_body(payload.clone()) + .create_async() + .await; + + let resp = reqwest::get(format!("{}/ipfs/bafkreibig", server.url())) + .await + .unwrap(); + let mut sink: Vec = Vec::new(); + write_object_to(resp, &mut sink).await.unwrap(); + + assert_eq!( + sink.len(), + payload.len(), + "a streamed body must arrive whole" + ); + assert!(sink == payload, "a streamed body must arrive unaltered"); + assert!( + diag_text().contains("deadbeef"), + "the header diagnostics still go to stderr, got: {}", + diag_text() + ); + m.assert_async().await; + } + + /// `node_tail`'s partial-body arm: a body that arrived part-way and then failed. + /// The other three `(read_failed, msg.is_empty())` combinations were covered; this + /// one, the shape a real broken connection most often produces, was not, because + /// the existing fixture writes zero body bytes. It is also the only arm where + /// node-supplied partial text reaches the terminal. + /// + /// The listener promises 512 bytes, writes a few, and hangs up. The terminal must + /// carry BOTH what did arrive and the fact that the rest did not. + #[tokio::test] + async fn test_cmd_get_reports_partial_text_and_the_unfinished_read() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + reset_diag(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + let mut scratch = [0u8; 2048]; + let _ = sock.read(&mut scratch).await; + let _ = sock + .write_all( + b"HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\n\ + Content-Length: 512\r\nConnection: close\r\n\r\n\ + {\"error\":\"boom\",\"message\":\"half a sen", + ) + .await; + let _ = sock.flush().await; + }); + + let err = cmd_get( + "bafkreipartial".to_string(), + format!("http://{addr}"), + None, + None, + ) + .await + .expect_err("a 500 is an error whatever became of its body"); + let told = told(&err); + + assert!( + told.contains("half a sen"), + "the text that DID arrive must reach the caller, got: {told}" + ); + assert!( + told.contains("could not be read in full"), + "and it must be marked as unfinished, or partial node text reads as the \ + node's whole answer, got: {told}" + ); + } } From c82a70ab21a21192b2579693b7d7c13aeb8a4d20 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Thu, 13 Aug 2026 18:55:58 -0500 Subject: [PATCH 66/77] fix(core): put the redirect predicate behind a feature so core stays lean The dependency allowlist caught this: sharing the predicate pulled url, and behind it idna and the icu crates, into the closure of the one crate that is supposed to stay embeddable. Both clients that need the predicate already parse URLs, so they opt in and nothing else pays for it. The test dependency is not optional, so the origin matrix still runs under a bare cargo test -p gitlawb-core. Gating the module on the feature alone would have left those tests silently unbuilt, which is the failure the module is there to prevent. --- crates/git-remote-gitlawb/Cargo.toml | 2 +- crates/gitlawb-core/Cargo.toml | 8 +++++++- crates/gitlawb-core/src/lib.rs | 7 +++++++ crates/gl/Cargo.toml | 2 +- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/crates/git-remote-gitlawb/Cargo.toml b/crates/git-remote-gitlawb/Cargo.toml index b6b9e76c..b704329e 100644 --- a/crates/git-remote-gitlawb/Cargo.toml +++ b/crates/git-remote-gitlawb/Cargo.toml @@ -11,7 +11,7 @@ name = "git-remote-gitlawb" path = "src/main.rs" [dependencies] -gitlawb-core = { path = "../gitlawb-core" } +gitlawb-core = { path = "../gitlawb-core", features = ["redirect"] } anyhow = { workspace = true } reqwest = { workspace = true } tracing = { workspace = true } diff --git a/crates/gitlawb-core/Cargo.toml b/crates/gitlawb-core/Cargo.toml index d2b3c05b..67e9ed9c 100644 --- a/crates/gitlawb-core/Cargo.toml +++ b/crates/gitlawb-core/Cargo.toml @@ -22,7 +22,7 @@ multihash-codetable = { workspace = true } cid = { workspace = true } chrono = { workspace = true } uuid = { workspace = true } -url = { workspace = true } +url = { workspace = true, optional = true } zeroize = { version = "1", features = ["derive"] } pkcs8 = { version = "0.10", features = ["pem", "std"] } curve25519-dalek = "4" @@ -31,3 +31,9 @@ chacha20poly1305 = "0.10" [dev-dependencies] tokio = { workspace = true } +# Not optional here: the redirect matrix must run under a bare +# `cargo test -p gitlawb-core`, with no feature selected. +url = { workspace = true } + +[features] +redirect = ["dep:url"] diff --git a/crates/gitlawb-core/src/lib.rs b/crates/gitlawb-core/src/lib.rs index ae6564a9..d0edec0a 100644 --- a/crates/gitlawb-core/src/lib.rs +++ b/crates/gitlawb-core/src/lib.rs @@ -5,6 +5,13 @@ pub mod encrypt; pub mod error; pub mod http_sig; pub mod identity; +// `url` is the one dependency here that drags a tail (idna, then the icu +// crates), and gitlawb-core is allowlisted to stay embeddable. Every client that +// needs this predicate already parses URLs, so they opt in and nothing else +// pays. `test` is in the cfg so `cargo test -p gitlawb-core` still compiles and +// runs the matrix below with no feature selected; without it the tests would +// silently not run, which is the failure this module exists to prevent. +#[cfg(any(feature = "redirect", test))] pub mod redirect; pub mod sanitize; pub mod scan_token; diff --git a/crates/gl/Cargo.toml b/crates/gl/Cargo.toml index 3d7ddb9f..2b973a4c 100644 --- a/crates/gl/Cargo.toml +++ b/crates/gl/Cargo.toml @@ -11,7 +11,7 @@ name = "gl" path = "src/main.rs" [dependencies] -gitlawb-core = { path = "../gitlawb-core" } +gitlawb-core = { path = "../gitlawb-core", features = ["redirect"] } icaptcha-client = { path = "../icaptcha-client" } base64 = { workspace = true } tokio = { workspace = true } From 5c0d6f32522adea8f167f8e445a26377b6fe3963 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:10:44 -0500 Subject: [PATCH 67/77] fix(core): refuse a redirect that rewrites the signed request-target may_follow compared host, port and scheme only, so a same-origin hop that normalized a trailing slash or a query was followed. The signature binds @path as the client sent it and the node rebuilds it from the URI it received, so that hop left the signature covering a target nobody asked for and the read 401d. Add the request-target clause and repoint the seven matrix rows that rode on a path change, so each keeps pinning the host or port property it exists for rather than going false on the path alone. --- crates/gitlawb-core/src/redirect.rs | 91 +++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 18 deletions(-) diff --git a/crates/gitlawb-core/src/redirect.rs b/crates/gitlawb-core/src/redirect.rs index bfb5738c..22830ae7 100644 --- a/crates/gitlawb-core/src/redirect.rs +++ b/crates/gitlawb-core/src/redirect.rs @@ -9,6 +9,13 @@ //! as the caller anywhere until the clock-skew window closes. On a 307/308 the //! request body goes along with it, which for the remote helper is the pack. //! +//! That same missing authority component is why the predicate also pins the +//! request-target: `@path` is signed as the client sent it and verified as the node +//! received it, so a same-origin hop that rewrites the path or the query (a +//! trailing-slash or query normalization) makes the signature cover a target the +//! node never saw and the read 401s. Only a hop that re-issues the identical target, +//! an http-to-https upgrade being the one that matters in practice, is followed. +//! //! The decision lives here rather than in either client because the two used to //! disagree: `gl` was scoped to the origin while the remote helper, the binary that //! actually runs `git clone gitlawb://`, still ran reqwest's default and followed @@ -27,12 +34,25 @@ /// too or it permits one hop fewer than it says. pub const MAX_REDIRECTS: usize = 10; -/// Follow a redirect only when it stays on the origin that issued it. +/// Follow a redirect only when it stays on the origin that issued it AND re-issues +/// the identical request-target. +/// +/// `Policy::none()` would have been the simpler answer, but one same-origin redirect +/// shape is legitimate here: a node fronted by a proxy that upgrades http to https, +/// or otherwise re-issues the same path and query, so the policy is scoped rather +/// than switched off. +/// +/// Path and query must match exactly, and that is the request-target clause rather +/// than an origin one. `@path` is signed as the client sent it and verified as the +/// node received it, so a hop that rewrites either half leaves a signature covering +/// a target nobody asked for and the node answers 401. Refusing the hop turns a +/// confusing 401 into the 3xx that names what actually happened. One policy covers +/// signed and unsigned callers alike, for the same reason the predicate is shared: +/// two rules would drift. /// -/// `Policy::none()` would have been the simpler answer, but same-origin redirects -/// are legitimate here (a node fronted by a proxy that upgrades http to https, or -/// normalizes a trailing slash), so the policy is scoped to the origin rather than -/// switched off. +/// `Url::query` is `None` for `/a` and `Some("")` for `/a?`, and those are two +/// different request-targets on the node side too, so the comparison is strict and +/// needs no special case. /// /// Host and port must match exactly. Port is compared as `Url::port`, which is /// `None` for a scheme's default port, so http -> https on the same host compares @@ -46,8 +66,9 @@ pub const MAX_REDIRECTS: usize = 10; /// matrix pins it: a move to raw string comparison would silently lose it. pub fn may_follow(previous: &url::Url, next: &url::Url) -> bool { let same_origin = next.host_str() == previous.host_str() && next.port() == previous.port(); + let same_target = next.path() == previous.path() && next.query() == previous.query(); let downgraded = previous.scheme() == "https" && next.scheme() != "https"; - same_origin && !downgraded + same_origin && same_target && !downgraded } #[cfg(test)] @@ -63,24 +84,25 @@ mod tests { ( "http://node.example/a", "http://node.example/b", - true, - "same origin, different path", + false, + "same origin but the path changed: @path is signed as sent and verified as received", ), ( "http://node.example/a", "https://node.example/a", true, - "http to https on one host: both ports are the scheme default", + "http to https on one host with an identical request-target: both ports are \ + the scheme default, so the proxy upgrade is still followed", ), ( "https://node.example/a", "https://node.example/a/", - true, - "trailing-slash normalization", + false, + "trailing-slash normalization changes the request-target, so it is refused", ), ( "https://node.example:8443/a", - "https://node.example:8443/b", + "https://node.example:8443/a", true, "same explicit port", ), @@ -112,42 +134,75 @@ mod tests { // because anything here compares case-insensitively or decodes IDN. They // are the variants an attacker reaches for, so they are pinned: swapping // this predicate for a raw string comparison must break the suite. + // Each of these pairs an IDENTICAL path on both sides, deliberately. The + // request-target clause below would make every one of them false on the + // path alone, and a row that is false for two reasons has stopped pinning + // either. With the paths equal, the host or port comparison is the only + // thing left that can decide them. ( "https://node.example/a", - "https://NODE.EXAMPLE/b", + "https://NODE.EXAMPLE/a", true, "same host in a different case: parse lowercases it", ), ( "https://node.example/a", - "https://node.example./b", + "https://node.example./a", false, "a trailing dot is a different host to url, so the redirect is refused", ), ( "https://exämple.test/a", - "https://xn--exmple-cua.test/b", + "https://xn--exmple-cua.test/a", true, "unicode host and its punycode spelling are one host after parse", ), ( "https://node.example/a", - "https://user:pw@node.example/b", + "https://user:pw@node.example/a", true, "userinfo is not part of the origin: same host, still followed", ), ( "https://node.example/a", - "https://node.example@attacker.example/b", + "https://node.example@attacker.example/a", false, "the node's name smuggled into userinfo: the host is the attacker's", ), ( "https://node.example/a", - "https://attacker.example#node.example/b", + "https://attacker.example/a#node.example", false, "the node's name pushed into the fragment: the host is the attacker's", ), + // The request-target clause, both directions. `@path` is the only thing a + // gitlawb signature binds the request to, so a hop that rewrites it hands + // the node a signature over a target it never received. + ( + "https://node.example/a?x=1", + "https://node.example/a?x=1", + true, + "identical request-target: the same-origin hop that is still followed", + ), + ( + "https://node.example/a?x=1", + "https://node.example/a?x=2", + false, + "same path but a different query: the request-target covers the query too", + ), + ( + "https://node.example/a", + "https://node.example/a?x=1", + false, + "a query added where there was none", + ), + ( + "http://node.example/a", + "http://node.example/a?", + false, + "an empty query added where there was none: a missing query and an empty \ + one are different request-targets", + ), ]; for (previous, next, expected, why) in cases { assert_eq!( From 92bb55441d9177eabf3151d0c5746f24c945f863 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:10:52 -0500 Subject: [PATCH 68/77] test(clients): prove the refused redirect never reaches the node verifier The two same-origin follow tests drove exactly the path-changing hop the predicate now refuses, and their mocks only proved a target was reached, never that the signature verified there. Rewrite both into refusal tests and run the real gitlawb-core verification over the request the target actually received, so the recorded verdict is what the assertion speaks about. Each refusal is paired with a positive control that verifies, so an empty verdict slot is attributable to the refusal rather than to a dead harness. --- crates/git-remote-gitlawb/src/main.rs | 344 ++++++++++++++++++++++---- crates/gl/src/http.rs | 256 ++++++++++++++++++- 2 files changed, 545 insertions(+), 55 deletions(-) diff --git a/crates/git-remote-gitlawb/src/main.rs b/crates/git-remote-gitlawb/src/main.rs index ea2ca400..1f779664 100644 --- a/crates/git-remote-gitlawb/src/main.rs +++ b/crates/git-remote-gitlawb/src/main.rs @@ -1014,11 +1014,20 @@ mod tests { ); } - /// The other direction: a same-origin redirect is still followed, so a node - /// fronted by a proxy that normalizes a path keeps working. Without this the - /// policy could be tightened to `Policy::none()` and nothing would notice. + /// A same-origin hop that REWRITES the request-target is refused, even though the + /// origin never changes. + /// + /// The signature binds `@path` as the literal path-and-query the helper signed, and + /// the node rebuilds it from the URI it received. An `info/refs` to `info/refs/` + /// bounce therefore presents a signature over a target the node never saw, so a + /// clone or push behind such a proxy 401s. Refusing the hop hands the caller the + /// 3xx that names what happened instead. + /// + /// The target mock expects zero hits and is asserted: mockito only checks + /// `.expect(N)` when `.assert()` runs, so an unbound or unasserted mock passes + /// vacuously. #[test] - fn a_same_origin_redirect_is_still_followed() { + fn a_same_origin_path_changing_redirect_is_refused() { let kp = Keypair::generate(); let client = build_http_client().unwrap(); @@ -1032,8 +1041,8 @@ mod tests { let target = node .mock("GET", "/zOwner/myrepo/info/refs/") .with_status(200) - .with_body("normalized") - .expect(1) + .with_body("bytes from the rewritten target") + .expect(0) .create(); let refs_url = format!("{}/zOwner/myrepo/info/refs", node.url()); @@ -1041,12 +1050,289 @@ mod tests { .send() .unwrap(); - assert_eq!(resp.status(), 200); - assert_eq!(resp.text().unwrap(), "normalized"); + assert_eq!( + resp.status(), + 301, + "a refused redirect stops rather than errors, so the caller sees the 3xx" + ); + assert!( + !resp + .text() + .unwrap() + .contains("bytes from the rewritten target"), + "the rewritten target's bytes must never reach the caller" + ); + bounce.assert(); + target.assert(); + } + + /// The other direction, and the only same-origin hop still followed: a redirect + /// that re-issues the IDENTICAL request-target. + /// + /// Without this the policy could be tightened to `Policy::none()` and nothing in + /// this crate would notice. It is also the first executed coverage of the chain + /// bound in `build_http_client`'s policy closure: the route answers 301 pointing at + /// itself, so bounded it is hit once for the original request plus `MAX_REDIRECTS` + /// follows and the call returns the 301 (a refused hop stops rather than errors). + /// Unbounded it would spin until `HTTP_TIMEOUT`, costing the node a request per + /// round trip. Its twin is `a_self_redirect_stops_at_the_chain_bound` in + /// `crates/gl/src/http.rs`. + #[test] + fn an_identical_target_redirect_is_still_followed_up_to_the_chain_bound() { + let kp = Keypair::generate(); + let client = build_http_client().unwrap(); + + let mut node = mockito::Server::new(); + let loop_route = node + .mock("GET", "/zOwner/myrepo/info/refs") + .with_status(301) + .with_header("location", "/zOwner/myrepo/info/refs") + .expect(gitlawb_core::redirect::MAX_REDIRECTS + 1) + .create(); + + let refs_url = format!("{}/zOwner/myrepo/info/refs", node.url()); + let resp = build_advertisement_request(&client, &refs_url, Some(&kp)) + .send() + .expect("the bound must end the chain, not the timeout"); + + assert_eq!( + resp.status(), + 301, + "the chain ends by refusing the next hop, so the last 3xx is what comes back" + ); + loop_route.assert(); + } + + // ── the node's own verification, run over what the helper actually sent ── + + /// What the verifying mock made of a request that reached it. + /// + /// The empty slot (`None`) is its own state and means the mock was never hit at + /// all, which is what the refusal test asserts. It must stay distinguishable from + /// [`Verdict::WrongIdentity`], because "nobody verified anything" and "something + /// verified against the wrong key" are opposite findings. + /// + /// The payloads are read through `Debug` in the assertion messages and nowhere + /// else, which the dead-code pass does not count; they carry the detail that makes + /// a failure legible, so they stay. + #[derive(Debug)] + #[allow(dead_code)] + enum Verdict { + /// The chain accepted the signature AND the key it resolved is the test's DID. + Accepted, + /// The chain refused it. Carries the error so a failure reads as the actual + /// rejection rather than a bare hit count. + Rejected(String), + /// The chain accepted a signature made by somebody else. A key resolved from + /// the parsed `key_id` is read out of the artifact under verification, so an + /// accept on it alone proves consistency, never authenticity. + WrongIdentity { expected: String, got: String }, + } + + /// The node's `require_signature` verification, over a request this crate did not + /// necessarily build: parse the headers, recompute the content-digest from the + /// body, rebuild the signing string over `@method`/`@path`/`content-digest`, + /// Ed25519-verify. Returns the DID the signature resolved to, so a caller can pin + /// the identity. + /// + /// Its twin is the hand-copy in `crates/gl/src/http.rs`, which cannot import this + /// module (this is a binary crate's test module). Keep the two textually identical + /// apart from the mockito seam around them, so an edit to one is visibly an edit to + /// both. + /// + /// It asserts internally, which is deliberate but constrains its callers: inside + /// `with_body_from_request` those assertions fire on the server thread and reach + /// the client as a transport error, not as a recorded verdict. So the identity + /// check lives in the caller as a [`Verdict`] variant, never as an assert in here. + fn node_verifies( + method: &str, + path_and_query: &str, + body: &[u8], + sig_input: &str, + sig_header: &str, + content_digest: &str, + ) -> anyhow::Result { + use gitlawb_core::http_sig::{build_signing_string, compute_content_digest, HttpSignature}; + use gitlawb_core::identity::verify; + use std::collections::HashMap; + + let sig = HttpSignature::parse(sig_input, sig_header)?; + sig.check_created()?; + assert!( + sig.missing_components().is_empty(), + "signature must cover all required components" + ); + assert_eq!(sig.alg, "ed25519"); + assert_eq!( + content_digest, + compute_content_digest(body), + "content-digest must match the body" + ); + let vk = sig.key_id.to_verifying_key()?; + let mut values = HashMap::new(); + values.insert("@method".to_string(), method.to_uppercase()); + values.insert("@path".to_string(), path_and_query.to_string()); + values.insert("content-digest".to_string(), content_digest.to_string()); + let sig_params_value = sig_input.strip_prefix("sig1=").unwrap_or(sig_input); + let components: Vec<&str> = sig.components.iter().map(String::as_str).collect(); + let signing_string = build_signing_string(&components, sig_params_value, &values)?; + let sig_array: [u8; 64] = sig.signature_bytes.as_slice().try_into()?; + verify(&vk, signing_string.as_bytes(), &sig_array)?; + Ok(sig.key_id.to_string()) + } + + /// Pull a header value off a received mockito request, or explain which one the + /// helper failed to send. + fn received_header(req: &mockito::Request, name: &str) -> String { + req.header(name) + .first() + .unwrap_or_else(|| panic!("the helper sent no {name} header")) + .to_str() + .unwrap() + .to_string() + } + + /// Run [`node_verifies`] over a GET that arrived at the mock and record what the + /// node would have made of it, pinned to `expected_did`. + fn record_get_verdict( + req: &mockito::Request, + expected_did: &str, + slot: &std::sync::Arc>>, + ) { + let verdict = match node_verifies( + "GET", + req.path_and_query(), + b"", + &received_header(req, "signature-input"), + &received_header(req, "signature"), + &received_header(req, "content-digest"), + ) { + Ok(did) if did == expected_did => Verdict::Accepted, + Ok(did) => Verdict::WrongIdentity { + expected: expected_did.to_string(), + got: did, + }, + Err(e) => Verdict::Rejected(e.to_string()), + }; + *slot.lock().unwrap() = Some(verdict); + } + + /// The finding's repro, now a guard: a rewritten same-origin target must never + /// receive the advertisement's signature, and the proof is the node's own + /// verification, not a hit count. + /// + /// Post-fix the hop is refused, so the slot stays empty. Pre-fix the hop is + /// followed and the slot records the Ed25519 rejection of a signature made over + /// `.../info/refs?service=git-upload-pack` and presented at `.../info/refs/...`, + /// which is the 401 an operator behind such a proxy actually sees. The verdict is + /// asserted first, so a failure speaks about verification rather than about + /// reachability. + /// + /// Its paired positive control is + /// `a_direct_signed_advertisement_verifies_under_the_node_verifier`: without it, an + /// empty slot would be satisfied just as well by a harness that can never record + /// anything. + #[test] + fn a_rewritten_target_never_receives_the_signature() { + let kp = Keypair::generate(); + let expected_did = kp.did().to_string(); + let client = build_http_client().unwrap(); + let slot = std::sync::Arc::new(std::sync::Mutex::new(None::)); + + let mut node = mockito::Server::new(); + let bounce = node + .mock("GET", "/zOwner/myrepo/info/refs?service=git-upload-pack") + .with_status(301) + .with_header( + "location", + "/zOwner/myrepo/info/refs/?service=git-upload-pack", + ) + .expect(1) + .create(); + let recorder = slot.clone(); + let did_for_target = expected_did.clone(); + let target = node + .mock("GET", "/zOwner/myrepo/info/refs/?service=git-upload-pack") + .with_status(200) + .with_body_from_request(move |req| { + record_get_verdict(req, &did_for_target, &recorder); + Vec::new() + }) + .expect(0) + .create(); + + let refs_url = format!( + "{}/zOwner/myrepo/info/refs?service=git-upload-pack", + node.url() + ); + let resp = build_advertisement_request(&client, &refs_url, Some(&kp)) + .send() + .unwrap(); + + let verdict = slot.lock().unwrap().take(); + assert!( + verdict.is_none(), + "the node's own verifier must never see this request: the signature covers \ + /zOwner/myrepo/info/refs?service=git-upload-pack and the rewritten target \ + adds a trailing slash, so what arrives there is a stale request-target; \ + recorded verdict: {verdict:?}" + ); + assert_eq!( + resp.status(), + 301, + "the caller sees the 3xx, not the rewritten target's answer" + ); bounce.assert(); target.assert(); } + /// The positive control for the test above, and the proof that the helper signs the + /// query it sends. + /// + /// A direct signed advertisement GET, no redirect anywhere, through the same + /// verifying mock. The verdict must be `Accepted`, which is what makes the refusal + /// test's empty slot attributable to the refusal rather than to a harness that + /// cannot record. The advertisement URL carries `?service=`, so a helper that + /// signed the bare path would land here as `Rejected`. + #[test] + fn a_direct_signed_advertisement_verifies_under_the_node_verifier() { + let kp = Keypair::generate(); + let expected_did = kp.did().to_string(); + let client = build_http_client().unwrap(); + let slot = std::sync::Arc::new(std::sync::Mutex::new(None::)); + + let mut node = mockito::Server::new(); + let recorder = slot.clone(); + let did_for_target = expected_did.clone(); + let route = node + .mock("GET", "/zOwner/myrepo/info/refs?service=git-upload-pack") + .with_status(200) + .with_body_from_request(move |req| { + record_get_verdict(req, &did_for_target, &recorder); + Vec::new() + }) + .expect(1) + .create(); + + let refs_url = format!( + "{}/zOwner/myrepo/info/refs?service=git-upload-pack", + node.url() + ); + let resp = build_advertisement_request(&client, &refs_url, Some(&kp)) + .send() + .unwrap(); + assert_eq!(resp.status(), 200); + + let verdict = slot.lock().unwrap().take(); + assert!( + matches!(verdict, Some(Verdict::Accepted)), + "a direct signed advertisement must verify under the node's own chain and \ + resolve to {expected_did}, or the refusal test's empty slot proves \ + nothing; recorded verdict: {verdict:?}" + ); + route.assert(); + } + /// The regression that round-1 missed: the Phase-2 `git-upload-pack` POST was /// left unsigned, so an owner's fetch of a private repo cleared the (now signed) /// advertisement and then 404'd on the pack POST. Drive BOTH request builders @@ -1326,49 +1612,13 @@ mod tests { /// to end (sign here, verify with the node's verifier), not reasoned. #[test] fn client_signature_verifies_under_node_verification_for_both_services() { - use gitlawb_core::http_sig::{build_signing_string, compute_content_digest, HttpSignature}; - use gitlawb_core::identity::verify; - use std::collections::HashMap; - let kp = Keypair::generate(); let client = reqwest::blocking::Client::new(); let body = b"0009done\n".to_vec(); - // Re-implements the node's require_signature verification (auth/mod.rs): - // parse headers, recompute content-digest from the body, rebuild the signing - // string over @method/@path/content-digest, Ed25519-verify. Ok iff the node - // would accept it. - let node_verifies = |method: &str, - path_and_query: &str, - body: &[u8], - sig_input: &str, - sig_header: &str, - content_digest: &str| - -> anyhow::Result<()> { - let sig = HttpSignature::parse(sig_input, sig_header)?; - sig.check_created()?; - assert!( - sig.missing_components().is_empty(), - "signature must cover all required components" - ); - assert_eq!(sig.alg, "ed25519"); - assert_eq!( - content_digest, - compute_content_digest(body), - "content-digest must match the body" - ); - let vk = sig.key_id.to_verifying_key()?; - let mut values = HashMap::new(); - values.insert("@method".to_string(), method.to_uppercase()); - values.insert("@path".to_string(), path_and_query.to_string()); - values.insert("content-digest".to_string(), content_digest.to_string()); - let sig_params_value = sig_input.strip_prefix("sig1=").unwrap_or(sig_input); - let components: Vec<&str> = sig.components.iter().map(String::as_str).collect(); - let signing_string = build_signing_string(&components, sig_params_value, &values)?; - let sig_array: [u8; 64] = sig.signature_bytes.as_slice().try_into()?; - verify(&vk, signing_string.as_bytes(), &sig_array)?; - Ok(()) - }; + // The verification chain is [`node_verifies`], the test-module helper the + // redirect verdict tests share. Same primitives the node's require_signature + // runs, over the request-target this crate transmits. // @path exactly as the node reconstructs it from the request it receives. let path_and_query = |req: &reqwest::blocking::Request| match req.url().query() { Some(q) => format!("{}?{}", req.url().path(), q), diff --git a/crates/gl/src/http.rs b/crates/gl/src/http.rs index 9a4614f9..7522c61d 100644 --- a/crates/gl/src/http.rs +++ b/crates/gl/src/http.rs @@ -780,6 +780,12 @@ mod tests { /// every time and only the chain bound ends the loop. Deleting the bound left the /// whole suite green, because nothing here had ever built a cycle. /// + /// It has a second job now. A redirect back to the identical path and query is the + /// only same-origin hop the predicate still follows, so the eleven hits below are + /// also this crate's executed proof that such a hop IS followed. The old positive + /// fixture drove a trailing-slash rewrite, which the request-target rule refuses, + /// and an http-to-https upgrade cannot be mocked over mockito's plain http. + /// /// The route answers 301 pointing back at itself. Bounded, the handler is hit /// once for the original request plus `MAX_REDIRECTS` follows and the call returns /// the 301 (a refused redirect stops rather than errors). Unbounded, it runs until @@ -828,11 +834,20 @@ mod tests { loop_route.assert_async().await; } - /// The other direction: a same-origin redirect is still followed, so a node - /// fronted by a proxy that normalizes a path keeps working. Without this the - /// policy could be tightened to `Policy::none()` and nothing would notice. + /// A same-origin hop that REWRITES the request-target is refused, even though the + /// origin never changes. + /// + /// The signature binds `@path` as the literal path-and-query the client signed, and + /// the node rebuilds it from the URI it received. A `/api/v1/thing` to + /// `/api/v1/thing/` bounce therefore presents a signature over a target the node + /// never saw, and every signed read behind such a proxy 401s. Refusing the hop + /// hands the caller the 3xx that names what happened instead. + /// + /// The target mock expects zero hits and is asserted: mockito only checks + /// `.expect(N)` when `.assert()` runs, so an unbound or unasserted mock passes + /// vacuously. #[tokio::test] - async fn same_origin_redirect_is_followed() { + async fn same_origin_path_changing_redirect_is_refused() { let mut node = Server::new_async().await; let bounce = node .mock("GET", "/api/v1/thing") @@ -844,20 +859,245 @@ mod tests { let target = node .mock("GET", "/api/v1/thing/") .with_status(200) - .with_body("normalized") - .expect(1) + .with_body("bytes from the rewritten target") + .expect(0) .create_async() .await; let client = NodeClient::new(node.url(), Some(test_keypair())); let resp = client.get_signed("/api/v1/thing").await.unwrap(); - assert_eq!(resp.status(), 200); - assert_eq!(resp.text().await.unwrap(), "normalized"); + assert_eq!( + resp.status(), + 301, + "a refused redirect stops rather than errors, so the caller sees the 3xx \ + and reports it through the status path it already has" + ); + let body = resp.text().await.unwrap(); + assert!( + !body.contains("bytes from the rewritten target"), + "the rewritten target's bytes must never reach the caller, got: {body}" + ); + bounce.assert_async().await; + target.assert_async().await; + } + + // ── the node's own verification, run over what the client actually sent ── + + /// What the verifying mock made of a request that reached it. + /// + /// The empty slot (`None`) is its own state and means the mock was never hit at + /// all, which is what the refusal test asserts. It must stay distinguishable from + /// [`Verdict::WrongIdentity`], because "nobody verified anything" and "something + /// verified against the wrong key" are opposite findings. + /// + /// The payloads are read through `Debug` in the assertion messages and nowhere + /// else, which the dead-code pass does not count; they carry the detail that makes + /// a failure legible, so they stay. + #[derive(Debug)] + #[allow(dead_code)] + enum Verdict { + /// The chain accepted the signature AND the key it resolved is the test's DID. + Accepted, + /// The chain refused it. Carries the error so a failure reads as the actual + /// rejection rather than a bare hit count. + Rejected(String), + /// The chain accepted a signature made by somebody else. A key resolved from + /// the parsed `key_id` is read out of the artifact under verification, so an + /// accept on it alone proves consistency, never authenticity. + WrongIdentity { expected: String, got: String }, + } + + /// The node's `require_signature` verification, over a request this crate did not + /// build: parse the headers, recompute the content-digest from the body, rebuild + /// the signing string over `@method`/`@path`/`content-digest`, Ed25519-verify. + /// Returns the DID the signature resolved to, so a caller can pin the identity. + /// + /// A hand-copy of its twin in `crates/git-remote-gitlawb/src/main.rs`, which gl + /// cannot import (`git-remote-gitlawb` is a binary crate and this is its test + /// module). Keep the two textually identical apart from the mockito seam around + /// them, so an edit to one is visibly an edit to both. + /// + /// It asserts internally, which is deliberate but constrains its callers: inside + /// `with_body_from_request` those assertions fire on the server thread and reach + /// the client as a transport error, not as a recorded verdict. So the identity + /// check lives in the caller as a [`Verdict`] variant, never as an assert in here. + fn node_verifies( + method: &str, + path_and_query: &str, + body: &[u8], + sig_input: &str, + sig_header: &str, + content_digest: &str, + ) -> anyhow::Result { + use gitlawb_core::http_sig::{build_signing_string, compute_content_digest, HttpSignature}; + use gitlawb_core::identity::verify; + use std::collections::HashMap; + + let sig = HttpSignature::parse(sig_input, sig_header)?; + sig.check_created()?; + assert!( + sig.missing_components().is_empty(), + "signature must cover all required components" + ); + assert_eq!(sig.alg, "ed25519"); + assert_eq!( + content_digest, + compute_content_digest(body), + "content-digest must match the body" + ); + let vk = sig.key_id.to_verifying_key()?; + let mut values = HashMap::new(); + values.insert("@method".to_string(), method.to_uppercase()); + values.insert("@path".to_string(), path_and_query.to_string()); + values.insert("content-digest".to_string(), content_digest.to_string()); + let sig_params_value = sig_input.strip_prefix("sig1=").unwrap_or(sig_input); + let components: Vec<&str> = sig.components.iter().map(String::as_str).collect(); + let signing_string = build_signing_string(&components, sig_params_value, &values)?; + let sig_array: [u8; 64] = sig.signature_bytes.as_slice().try_into()?; + verify(&vk, signing_string.as_bytes(), &sig_array)?; + Ok(sig.key_id.to_string()) + } + + /// Pull a header value off a received mockito request, or explain which one the + /// client failed to send. + fn received_header(req: &mockito::Request, name: &str) -> String { + req.header(name) + .first() + .unwrap_or_else(|| panic!("the client sent no {name} header")) + .to_str() + .unwrap() + .to_string() + } + + /// Run [`node_verifies`] over a GET that arrived at the mock and record what the + /// node would have made of it, pinned to `expected_did`. + fn record_get_verdict( + req: &mockito::Request, + expected_did: &str, + slot: &std::sync::Arc>>, + ) { + let verdict = match node_verifies( + "GET", + req.path_and_query(), + b"", + &received_header(req, "signature-input"), + &received_header(req, "signature"), + &received_header(req, "content-digest"), + ) { + Ok(did) if did == expected_did => Verdict::Accepted, + Ok(did) => Verdict::WrongIdentity { + expected: expected_did.to_string(), + got: did, + }, + Err(e) => Verdict::Rejected(e.to_string()), + }; + *slot.lock().unwrap() = Some(verdict); + } + + /// The finding's repro, now a guard: a rewritten same-origin target must never + /// receive the signature, and the proof is the node's own verification, not a hit + /// count. + /// + /// The target mock runs the full `require_signature` chain over the request it + /// receives. Post-fix the hop is refused, so the slot stays empty. Pre-fix the hop + /// is followed and the slot records the Ed25519 rejection of a signature made over + /// `/api/v1/thing` and presented at `/api/v1/thing/`, which is the 401 an operator + /// behind such a proxy actually sees. The verdict is asserted first, so a failure + /// speaks about verification rather than about reachability. + /// + /// Its paired positive control is + /// `a_direct_signed_get_verifies_under_the_node_verifier`: without it, an empty + /// slot would be satisfied just as well by a harness that can never record + /// anything. + #[tokio::test] + async fn a_rewritten_target_never_receives_the_signature() { + let kp = test_keypair(); + let expected_did = kp.did().to_string(); + let slot = std::sync::Arc::new(std::sync::Mutex::new(None::)); + + let mut node = Server::new_async().await; + let bounce = node + .mock("GET", "/api/v1/thing") + .with_status(301) + .with_header("location", "/api/v1/thing/") + .expect(1) + .create_async() + .await; + let recorder = slot.clone(); + let did_for_target = expected_did.clone(); + let target = node + .mock("GET", "/api/v1/thing/") + .with_status(200) + .with_body_from_request(move |req| { + record_get_verdict(req, &did_for_target, &recorder); + Vec::new() + }) + .expect(0) + .create_async() + .await; + + let client = NodeClient::new(node.url(), Some(kp)); + let resp = client.get_signed("/api/v1/thing").await.unwrap(); + + let verdict = slot.lock().unwrap().take(); + assert!( + verdict.is_none(), + "the node's own verifier must never see this request: the signature covers \ + /api/v1/thing and the rewritten target is /api/v1/thing/, so what arrives \ + there is a stale request-target; recorded verdict: {verdict:?}" + ); + assert_eq!( + resp.status(), + 301, + "the caller sees the 3xx, not the rewritten target's answer" + ); bounce.assert_async().await; target.assert_async().await; } + /// The positive control for the test above, and the proof that the client signs + /// the query it sends. + /// + /// A direct signed GET, no redirect anywhere, through the same verifying mock. The + /// verdict must be `Accepted`, which is what makes the refusal test's empty slot + /// attributable to the refusal rather than to a harness that cannot record. The + /// path carries a query, so a client that signed the bare path would land here as + /// `Rejected`. + #[tokio::test] + async fn a_direct_signed_get_verifies_under_the_node_verifier() { + let kp = test_keypair(); + let expected_did = kp.did().to_string(); + let slot = std::sync::Arc::new(std::sync::Mutex::new(None::)); + + let mut node = Server::new_async().await; + let recorder = slot.clone(); + let did_for_target = expected_did.clone(); + let route = node + .mock("GET", "/api/v1/thing?x=1") + .with_status(200) + .with_body_from_request(move |req| { + record_get_verdict(req, &did_for_target, &recorder); + Vec::new() + }) + .expect(1) + .create_async() + .await; + + let client = NodeClient::new(node.url(), Some(kp)); + let resp = client.get_signed("/api/v1/thing?x=1").await.unwrap(); + assert_eq!(resp.status(), 200); + + let verdict = slot.lock().unwrap().take(); + assert!( + matches!(verdict, Some(Verdict::Accepted)), + "a direct signed GET must verify under the node's own chain and resolve to \ + {expected_did}, or the refusal test's empty slot proves nothing; recorded \ + verdict: {verdict:?}" + ); + route.assert_async().await; + } + // ── read_body_capped ──────────────────────────────────────────────── /// A body whose read FAILS mid-stream must be distinguishable from a body that From 1c86ee22d65412c98d50031192772658bbea4224 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:48:46 -0500 Subject: [PATCH 69/77] test(node): pin require_signature to the request-target it received The verifier already rebuilds @path from the URI it was sent, which is why the client-side redirect bug surfaced as a 401 rather than as a bypass. Nothing pinned that, so a change to the reconstruction could drop the query half or collapse it to a constant and only the clients would notice. Drive the production router fixture with a signature made over one path and a request on another, and again with a query mismatch, plus an identically signed control. No pre-fix RED is obtainable here by construction, so each case is proven load-bearing by injecting the defect it names. --- crates/gitlawb-node/src/test_support.rs | 121 ++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 785ebf35..109e94d6 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -1962,6 +1962,127 @@ mod tests { ); } + /// A signed request body reused by the request-target trio below. Non-empty so + /// the content-digest the signature covers is a real hash rather than the + /// empty-body constant, which keeps `@path` the only component under test. + const TARGET_PIN_BODY: &[u8] = br#"{"task_type":"noop","payload":{}}"#; + + /// Send `body` to `uri` carrying a signature made over `signed_over`, through + /// the PRODUCTION router (`app`, which goes through `server::build_router`, where + /// `add_auth_layers` installs `require_signature` on the write routes). Returns + /// the status and the parsed JSON body (`Null` when the response is not JSON, as + /// a handler response past the middleware may be). Going through `app` rather + /// than a hand-mounted `Router::new().route(...)` probe is the point: a bare + /// router answers whether the middleware rejects the request, not whether that + /// is how a caller is actually gated. + async fn signed_over_then_sent( + pool: PgPool, + signed_over: &str, + uri: &str, + ) -> (StatusCode, serde_json::Value) { + use gitlawb_core::http_sig::sign_request; + use gitlawb_core::identity::Keypair; + + let kp = Keypair::generate(); + let signed = sign_request(&kp, "POST", signed_over, TARGET_PIN_BODY); + let req = Request::builder() + .method(Method::POST) + .uri(uri) + .header(axum::http::header::CONTENT_TYPE, "application/json") + .header("content-digest", signed.content_digest) + .header("signature-input", signed.signature_input) + .header("signature", signed.signature) + .body(Body::from(TARGET_PIN_BODY)) + .unwrap(); + + let resp = app(pool).await.oneshot(req).await.unwrap(); + let status = resp.status(); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let json = serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null); + (status, json) + } + + /// The server half of the redirect finding: `require_signature` rebuilds `@path` + /// from the URI of the request it actually received, so a signature minted over + /// `/api/v1/repos` and presented at `POST /api/v1/tasks` verifies against the + /// wrong request-target and is refused 401 `invalid_signature`. Both routes sit + /// behind `add_auth_layers` in `build_router`, so the request really does reach + /// the middleware instead of 404ing at the fallback. This is the node-side proof + /// that a client which lets a redirect rewrite the target gets a 401, which is + /// what the production report showed. + /// + /// No pre-fix RED is obtainable here: the verifier already gates on `@path` (that + /// is precisely why the client bug surfaced as a 401 rather than as a silently + /// accepted request), so there is no broken state to observe first. The test is a + /// must-not guard, green by design, and its RED proof is by mutation of the + /// reconstruction it pins. + #[sqlx::test] + async fn require_signature_refuses_a_stale_request_target_path(pool: PgPool) { + let (status, json) = signed_over_then_sent(pool, "/api/v1/repos", "/api/v1/tasks").await; + assert_eq!( + status, + StatusCode::UNAUTHORIZED, + "a signature minted over one route path must not verify when replayed on another" + ); + assert_eq!( + json["error"], "invalid_signature", + "the refusal must come from the signature check, not from a handler or a later gate" + ); + } + + /// The query half of the same reconstruction: `@path` is path-and-query, not path + /// alone, so a signature minted over `/api/v1/tasks` and sent to + /// `/api/v1/tasks?x=1` is refused 401 `invalid_signature` too. Without this case a + /// reconstruction narrowed to `parts.uri.path()` would keep the sibling test above + /// green while admitting every query rewrite, so both components of the received + /// target are pinned rather than just the first. + /// + /// No pre-fix RED is obtainable here either, for the reason given on the sibling + /// above: the verifier already covers the query, so this is a green-by-design + /// must-not guard whose RED proof is by mutation. + #[sqlx::test] + async fn require_signature_refuses_a_stale_request_target_query(pool: PgPool) { + let (status, json) = + signed_over_then_sent(pool, "/api/v1/tasks", "/api/v1/tasks?x=1").await; + assert_eq!( + status, + StatusCode::UNAUTHORIZED, + "a signature minted over the bare task path \ + must not verify on a request that carries a query" + ); + assert_eq!( + json["error"], "invalid_signature", + "the query refusal must come from the signature check, \ + not from a handler rejecting the unknown parameter" + ); + } + + /// The paired positive control the two refusals need to mean anything: signed and + /// sent over the identical target `/api/v1/tasks?x=1`, the request clears + /// `require_signature`. Without it a reconstruction that produced garbage for + /// every request would satisfy both refusals above and look like coverage. The + /// request carries no `x-ucan` header, so `require_ucan_chain` passes it through + /// and whatever status arrives past the auth pair is the handler's own; the + /// assertion is therefore that the response is NOT the 401 `invalid_signature` the + /// mismatch cases get, not a pin on some particular handler outcome. + /// + /// Green by design like its siblings, and for the same reason: the verifier + /// already reconstructs the received target, so there is no pre-fix RED to + /// observe and the proof that this assertion is load-bearing comes from degrading + /// the reconstruction under mutation. + #[sqlx::test] + async fn require_signature_admits_the_exact_request_target(pool: PgPool) { + let (status, json) = + signed_over_then_sent(pool, "/api/v1/tasks?x=1", "/api/v1/tasks?x=1").await; + assert!( + !(status == StatusCode::UNAUTHORIZED && json["error"] == "invalid_signature"), + "an identically signed and sent request-target must clear require_signature, \ + so this control must not draw the same refusal as the mismatch cases; got {status}" + ); + } + /// Issue #6 / jatmn finding 2: `/api/v1/stats` counts logical repos, not raw /// rows. With a mirror+canonical pair and a standalone repo present, the /// `repos` count is 2. From dc5daa14b4d0816d8a14fabdf057259c1d177a46 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:43:48 -0500 Subject: [PATCH 70/77] fix(node): size the ipfs work floor for provenance walks plus one full search The floor reserved one complete legacy search per window, probes plus page tolls, but the provenance visibility walk debits the same bucket before the fallback runs and its cap is charged per phase. With a route limit set below the floor the provenance phase spent from the budget the search was promised, the fallback 429d short of its configured reach, and the retry re-paid the same walk charges, so a readable holder past that point stayed unreachable. Add the walk term, min(MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_repos_walked), which is textually the resolver's own walk_cap so the two move together. The ladder fixture pins repos-walked to 1 to keep the page toll, not the new term, as the thing binding it. --- crates/gitlawb-node/src/api/ipfs.rs | 225 +++++++++++++++++++++++++++- crates/gitlawb-node/src/config.rs | 73 ++++++--- crates/gitlawb-node/src/state.rs | 40 ++++- 3 files changed, 301 insertions(+), 37 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 00188ee7..2c078371 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -2319,6 +2319,29 @@ mod tests { .expect("in-range stamp") } + /// Stamp an already-seeded repo row's `created_at` with [`scan_order_stamp`], for a + /// fixture whose RED depends on WHICH row the scan reaches last. + /// + /// `upsert_mirror_repo` (under `seed_repo_with_blob`) stamps `Utc::now()`, so a + /// mirror row's scan position is its seeding instant rendered by `to_rfc3339`, whose + /// fractional-second field is variable-width. The scan compares the stored TEXT, so + /// two rows seeded milliseconds apart can order by digit count rather than by time. + /// Restamping with the whole-second values keeps text order and seed order identical. + async fn stamp_scan_order(pool: &sqlx::PgPool, repo_id: &str, i: usize) { + let at = scan_order_stamp(i).to_rfc3339(); + let done = sqlx::query("UPDATE repos SET created_at = $1 WHERE id = $2") + .bind(&at) + .bind(repo_id) + .execute(pool) + .await + .expect("restamp a seeded repo's scan position"); + assert_eq!( + done.rows_affected(), + 1, + "restamping {repo_id} must hit exactly the row the fixture seeded" + ); + } + /// Seed `n` PRIVATE repos owned by a foreign DID, in scan order, with `rules_each` /// path-scoped rules apiece. An anonymous caller is denied at the root gate on every /// one, and a root deny costs neither a probe nor a visit, which is exactly the @@ -3103,6 +3126,187 @@ mod tests { assert_eq!(body["error"], "search_incomplete", "{body}"); } + /// F2 (#173 round 15): the derived work floor must fit ONE COMPLETE COMBINED + /// resolution, provenance walks included, not just the legacy search. + /// + /// `AppState::ipfs_work_budget` floors the per-IP work bucket at + /// `ipfs_max_legacy_probes + pages`, but the SAME bucket is debited once per + /// provenance visibility walk, before the fallback the markers arm has run at all. + /// So with `GITLAWB_IPFS_RATE_LIMIT` below the floor (the only configuration where + /// the floor is what sizes the bucket), the provenance phase eats into the budget + /// the floor exists to reserve for the search, and the "one complete legacy search + /// per window" guarantee stops holding: the search 429s short of its configured + /// reach, and the retry re-pays the same provenance charges. + /// + /// The seams, stated the way the sibling fixtures do, and the ledger they produce: + /// + /// * `ipfs_rate_limit = 1`, below the floor, so the floor is what binds. + /// * `ipfs_max_legacy_probes = 4`, above the three probes the scan spends, so the + /// probe ceiling is NOT what stops the holder (it is a second brake that can + /// strand it independently of the work bucket, which is why the GREEN is + /// asserted as a SERVED 200 rather than as merely not-429). + /// * `ipfs_max_legacy_scan_rows = 128`, one page at the production page size, so + /// the scan buys exactly one page toll. + /// * `ipfs_max_repos_walked = 2`, so `walk_cap` is `min(17, 2) = 2` and exactly + /// fits the two path-scoped provenance deniers per phase. + /// * `ipfs_max_repo_visits` stays at its 1024 default against the 5 visits here, + /// so no other ceiling binds. + /// + /// Debits, in order: 2 provenance walks (the `!legacy_scan` charge, one per denier, + /// with no probe toll on that phase), 1 page toll, then one probe per legacy + /// candidate. The two deniers are re-visited by the scan for free as far as WALKS + /// go (the allowed-set memo persists across phases) but each still pays its probe, + /// so the holder's own probe is the SIXTH debit. + /// + /// Old floor `4 + 1 = 5`: that sixth debit finds the bucket empty, + /// `gate_and_serve` returns `Throttled` WITHOUT tainting, and the tail renders the + /// work-path 429. New floor `4 + 1 + min(17, 2) = 7`: the holder is reached, + /// walked on the scan phase's own budget, and served, with one token to spare. + /// + /// The route limiter is deliberately left at `test_support`'s default rather than + /// sized from this cfg. `ipfs_router` layers no `rate_limit_by_ip` at all, so that + /// saves nothing today, but a route bucket sized from `ipfs_rate_limit = 1` would + /// shed the request at the door and the RED would be a 429-vs-429 collision with no + /// discriminant. For the same reason the RED assertion pins the "ipfs retrieval" + /// prefix: the route brake's body is "rate limit exceeded", a substring of the + /// work path's "ipfs retrieval rate limit exceeded", so a bare status check or the + /// shorter string cannot tell the two brakes apart. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_work_floor_fits_provenance_walks_plus_one_full_legacy_search( + pool: sqlx::PgPool, + ) { + use crate::state::AppState; + use clap::Parser; + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let cfg = crate::config::Config::parse_from([ + "gitlawb-node", + "--ipfs-rate-limit", + "1", + "--ipfs-max-legacy-probes", + "4", + "--ipfs-max-legacy-scan-rows", + "128", + "--ipfs-max-repos-walked", + "2", + ]); + // The knobs live in TWO places: `build_state` seeds the probe and scan-row + // ceilings the resolver enforces as AppState fields from constants, independent + // of Config, while `walk_cap` reads `state.config.ipfs_max_repos_walked`. A cfg + // installed without the seams would size the bucket from one set of values and + // run the scan under another. + state.ipfs_max_legacy_probes = AppState::ipfs_legacy_probe_budget(&cfg); + state.ipfs_max_legacy_scan_rows = AppState::ipfs_legacy_scan_row_budget(&cfg); + assert_eq!( + state.ipfs_legacy_scan_page_rows, + crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, + "fixture precondition: the page seam stays at the production page size, so \ + the row ceiling above is exactly one page and the scan buys one page toll" + ); + assert_eq!( + state.ipfs_max_history_walks, + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + "fixture precondition: the history-walk seam stays at the constant, so \ + walk_cap is min(17, 2) = 2 and the repos-walked knob is what binds" + ); + state.config = Arc::new(cfg.clone()); + // The bucket is sized from the seam under test, never by hand: that is what + // makes the floor change, and nothing else, the difference between RED and GREEN. + let floor = AppState::ipfs_work_budget(&cfg); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(floor, std::time::Duration::from_secs(3600)); + + // Identical content everywhere, so one CID resolves to one oid all three repos + // carry. Scan order is `(created_at, id)` ASC and the holder must be paged AFTER + // both deniers, or its probe is not the debit that finds the bucket empty. + let content = b"one complete combined resolution\n"; + let (prov_one, oid) = + seed_path_denying_repo(&state, tmp.path(), "z6f2floor", "provdeny-one", content).await; + let (prov_two, _) = + seed_path_denying_repo(&state, tmp.path(), "z6f2floor", "provdeny-two", content).await; + // The holder's rule IS path-scoped, so reaching its verdict still costs a walk, + // but it covers a path this object is not at, so the walk's allowed set decides + // on the mirror row's public flag and ALLOWS an anonymous reader. + let (holder_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f2floor", "holder", content).await; + state + .db + .set_visibility_rule( + &holder_id, + "/decoy/**", + crate::db::VisibilityMode::B, + &[OTHER_READER.to_string()], + "z6f2floor", + ) + .await + .unwrap(); + stamp_scan_order(&pool, &prov_one, 0).await; + stamp_scan_order(&pool, &prov_two, 1).await; + stamp_scan_order(&pool, &holder_id, 2).await; + + // The two deniers are the recorded sources; the holder is not, which is the + // dropped-source case. Two sources sits well under MAX_PIN_SOURCES (16), so + // `pin_sources_at_cap` cannot arm the fallback: the durable incomplete marker is + // what arms it. + state.db.record_pin_source(&oid, &prov_one).await.unwrap(); + state.db.record_pin_source(&oid, &prov_two).await.unwrap(); + state + .db + .mark_pin_sources_incomplete(&oid, "") + .await + .unwrap(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let work_bucket = state.ipfs_work_rate_limiter.clone(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.173:5000".parse().unwrap(); + let resp = router.oneshot(get_cid(&cid, Some(peer))).await.unwrap(); + let status = resp.status(); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let rendered = String::from_utf8_lossy(&body).to_string(); + // Drain what the request left, so the failure messages carry the measured + // ledger rather than only its consequence. + let mut spare = 0usize; + while work_bucket.check("203.0.113.173").await { + spare += 1; + } + + assert!( + !rendered.contains("ipfs retrieval"), + "the work floor must reserve a full legacy search AFTER the provenance \ + phase has taken its walks off the same bucket. The holder's own probe \ + found the bucket empty and the tail rendered the work-path 429 (floor \ + {floor}, {spare} of it unspent): {rendered}" + ); + assert_eq!( + status, + StatusCode::OK, + "the buried public holder must be SERVED within one window, not merely \ + spared the 429: the probe ceiling is a second brake that can strand it on \ + its own (floor {floor}, {spare} unspent): {rendered}" + ); + assert_eq!( + &body[..], + content.as_slice(), + "the served bytes must be the holder's object" + ); + assert_eq!( + spare, 1, + "the measured ledger is 2 provenance walks + 1 page toll + 3 legacy probes \ + = 6 debits against a floor of {floor}, so exactly one token is left. A \ + different remainder means the debit order moved and the RED above is no \ + longer pinned on the holder's probe" + ); + } + /// F2 visit ceiling: `ipfs_max_repo_visits` bounds the acquire+probe cost class /// (each visit can be a full Tigris archive fetch on a cache miss). Unlike the /// walk cap there is no cheap way to keep scanning, so exhaustion STOPS the scan @@ -4122,14 +4326,21 @@ mod tests { /// `ceil(10 / 4) + 1 = 4` requests. Every intermediate response is the retryable /// 503 with a token, and no 429 interrupts the ladder, which is what the floor fix /// pins. The work bucket is sized to the DERIVED floor of a config whose page term - /// dominates (probe knob 1, row knob 896 = 7 pages, so floor = 8); under the old - /// floor (`max(route, probes)` = 1) the very first page would 429. + /// dominates (probe knob 1, row knob 896 = 7 pages, walk knob 1, so floor = 9); + /// under the old floor (`max(route, probes)` = 1) the very first page would 429. /// - /// The floor is 8 rather than the honest ladder's exact cost (6 pages + 1 probe = 7) + /// The walk knob is pinned at 1 rather than left at its default of 64. This ladder + /// is a pure legacy scan with no provenance phase, so the floor's walk term + /// (`min(17, ipfs_max_repos_walked)`, #173 round 15) buys nothing the fixture + /// spends; at the default it would hand the bucket 17 tokens of slack and the page + /// toll, which is the thing this test exists to hold the floor against, would stop + /// being what binds. + /// + /// The floor is 9 rather than the honest ladder's exact cost (6 pages + 1 probe = 7) /// on purpose. A ladder that never resumes re-pages from the front every request and /// costs 8, so at a bucket of 7 mutation C would trip the 429 guard one step before /// the reach guard and its RED would be attributed to the toll rather than to the - /// missing continuation. One token of headroom keeps each guard reporting its own + /// missing continuation. A token of headroom keeps each guard reporting its own /// property. /// /// MUTATION C (RED): emit the token but never open it on the way in, and the ladder @@ -4167,11 +4378,13 @@ mod tests { "1", "--ipfs-max-legacy-scan-rows", "896", + "--ipfs-max-repos-walked", + "1", ]); let floor = AppState::ipfs_work_budget(&cfg); assert_eq!( - floor, 8, - "fixture precondition: 1 probe + 896/128 = 7 pages" + floor, 9, + "fixture precondition: 1 probe + 896/128 = 7 pages + min(17, 1) = 1 walk" ); state.ipfs_work_rate_limiter = crate::rate_limit::RateLimiter::new(floor, std::time::Duration::from_secs(3600)); diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 00ab6edd..8cecd526 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -1022,11 +1022,13 @@ mod tests { } /// The `/ipfs` work-budget capacity is DERIVED from the route limit (R6, KTD6), with - /// a hard floor of one full legacy search per window (the effective - /// `ipfs_max_legacy_probes`). This guards the derived default so a single - /// default-config deep search never self-throttles mid-scan and recreates the F6 - /// admit-then-429 for a legitimate caller. A `RateLimiter` sized to the derived - /// budget must admit the whole probe budget back to back. + /// a hard floor of one complete COMBINED resolution per window: the provenance + /// phase's walk term plus a full legacy search (the effective + /// `ipfs_max_legacy_probes` plus the row ceiling's page toll). This guards the + /// derived default so a single default-config deep search never self-throttles + /// mid-scan and recreates the F6 admit-then-429 for a legitimate caller. A + /// `RateLimiter` sized to the derived budget must admit the whole budget back to + /// back. #[test] fn ipfs_work_budget_derives_from_route_limit_and_clears_the_probe_floor() { use crate::state::AppState; @@ -1041,23 +1043,45 @@ mod tests { "the work budget must clear one full legacy search per window" ); - // Tight route limit (1): the floor lifts the work budget to a full deep scan, - // the 256-probe budget PLUS the page toll a 2048-row ceiling costs at 128 rows - // per page (16) = 272, NOT down to 1. A single deep search still completes its - // full scan without self-throttling on either charge. + // Tight route limit (1): the floor lifts the work budget to one complete + // COMBINED resolution, the 256-probe budget PLUS the page toll a 2048-row + // ceiling costs at 128 rows per page (16) PLUS the provenance phase's walk term + // min(17, 64) = 17, so 289, NOT down to 1. The provenance walks come off the + // same bucket before the fallback runs, so a floor without that term hands the + // legacy search a bucket the provenance phase already spent from. This case + // also carries the walk term's ABOVE-constant direction: the repos-walked knob + // is at its default 64, so `MAX_HISTORY_WALKS_PER_REQUEST` (17) is what binds. let tight = Config::parse_from(["gitlawb-node", "--ipfs-rate-limit", "1"]); assert_eq!( AppState::ipfs_work_budget(&tight), - 272, - "a tight route limit is floored at probes + pages (256 + 16), not clamped to 1" + 289, + "a tight route limit is floored at probes + pages + walks \ + (256 + 16 + min(17, 64) = 17), not clamped to 1" + ); + + // The walk term's BELOW-constant direction: a repos-walked knob under the + // history-walk constant is what the resolver's own `walk_cap` min() selects, so + // it is what the floor must carry too. 256 + 16 + min(17, 3) = 275. + let narrow_walk = Config::parse_from([ + "gitlawb-node", + "--ipfs-rate-limit", + "1", + "--ipfs-max-repos-walked", + "3", + ]); + assert_eq!( + AppState::ipfs_work_budget(&narrow_walk), + 275, + "the walk term takes min(17, repos-walked 3) = 3, the resolver's own \ + walk_cap, so the floor is 256 + 16 + 3" ); // Raised probe budget lifts the floor with it (the work budget tracks the - // effective probe budget, not the constant). The walk cap is set to a DIFFERENT - // value in the same config on purpose: the two were one field before the split, - // so a floor that silently read the walk cap would return 7 here and still look - // plausible. Only the legacy-probe and legacy-scan-rows knobs may drive this - // budget. + // effective probe budget, not the constant). The walk cap here is a SECOND + // below-constant proof at a different pair of values: min(17, 7) = 7, and the + // probe knob is raised at the same time so a floor that folded the two terms + // together (they were one field before the split) reads visibly wrong rather + // than plausibly right. let raised = Config::parse_from([ "gitlawb-node", "--ipfs-rate-limit", @@ -1069,16 +1093,17 @@ mod tests { ]); assert_eq!( AppState::ipfs_work_budget(&raised), - 1016, + 1023, "the floor tracks the operator-raised legacy-probe budget (1000) plus the \ - default row ceiling's page toll (16), not the walk cap" + default row ceiling's page toll (16) plus the walk term min(17, 7) = 7" ); // The scan-rows knob is coupled to the floor too, and this EXECUTES the coupling // rather than describing it: every page the ceiling permits is charged to the // caller's work bucket, so a raised ceiling that did not lift the floor would // 429 an honest caller part-way down their own token ladder. 4096 rows at 128 - // rows per page is 32 pages, so the floor is 256 + 32. + // rows per page is 32 pages, so the floor is 256 + 32 + the default walk term + // of 17. let wide_scan = Config::parse_from([ "gitlawb-node", "--ipfs-rate-limit", @@ -1088,9 +1113,10 @@ mod tests { ]); assert_eq!( AppState::ipfs_work_budget(&wide_scan), - 288, + 305, "raising the row ceiling must raise the work floor by the pages it buys \ - (256 probes + 4096/128 = 32 pages), or a full deep scan self-throttles" + (256 probes + 4096/128 = 32 pages + min(17, 64) = 17 walks), or a full \ + deep scan self-throttles" ); // 0 route limit disables the derived bucket too (a 0-capacity limiter admits all). @@ -1102,7 +1128,8 @@ mod tests { ); // Behavioral floor: a limiter sized to the derived (tight-route) budget admits - // the whole probe budget back to back for one source, then sheds the next. + // a whole combined resolution's worth of charges back to back for one source, + // then sheds the next. let budget = AppState::ipfs_work_budget(&tight); let limiter = crate::rate_limit::RateLimiter::new(budget, std::time::Duration::from_secs(3600)); @@ -1114,7 +1141,7 @@ mod tests { for i in 0..budget { assert!( limiter.check("1.2.3.4").await, - "probe {i} of one full default-config scan must be admitted (no mid-scan throttle)" + "charge {i} of one full combined resolution must be admitted (no mid-scan throttle)" ); } assert!( diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 85356fb4..3608089f 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -429,16 +429,36 @@ impl AppState { /// (R6, KTD6), DERIVED from the route limit rather than a new operator knob. The route /// limiter (`ipfs_rate_limiter`) charges once per request; this separate bucket absorbs /// the resolver's per-probe/per-walk work charges so both the route "requests per hour" - /// contract and the amplification bound hold. Floor: at least one full legacy search per - /// window, the effective `ipfs_max_legacy_probes` (the `GITLAWB_IPFS_MAX_LEGACY_PROBES` - /// knob), so a single default-config deep search cannot self-throttle mid-scan and - /// recreate the admit-then-429 for a legitimate caller. `GITLAWB_IPFS_RATE_LIMIT=0` + /// contract and the amplification bound hold. Floor: at least one complete COMBINED + /// resolution per window, the provenance phase's walks plus a full legacy search + /// (probes plus page tolls), so a single default-config resolution cannot + /// self-throttle part-way and recreate the admit-then-429 for a legitimate caller. + /// `GITLAWB_IPFS_RATE_LIMIT=0` /// disables the route brake and this derived bucket alike (a 0-capacity limiter admits /// everything). /// - /// The floor is the LEGACY-PROBE knob, not `ipfs_max_repos_walked`. Those were one - /// field before the walk cap and the probe budget were split apart, and reading the - /// walk cap here would silently size this bucket at 64 instead of 256. + /// The floor carries a WALK term (#173 round 15, F2). The provenance visibility walk + /// in `gate_and_serve` debits this SAME bucket (its `!legacy_scan` charge), once per + /// path-scoped source, before the legacy fallback runs at all, and the walk cap is + /// charged per phase. A floor counting only the search therefore under-sizes the + /// window by up to that cap exactly when the floor binds (a route limit set below + /// it): the provenance phase spends from the budget the floor reserved for the + /// search, the fallback 429s short of its configured reach, and the retry re-pays the + /// same provenance charges. The term is + /// `min(MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_repos_walked)` because that is + /// textually the resolver's own `walk_cap` in `gate_and_serve`; the two `min()`s must + /// move together, so an edit to either finds the other from here. It reads the + /// CONSTANT and the CONFIG knob for the same reason the page term below reads the + /// constant page size: `ipfs_max_history_walks` is an `AppState` test seam, and + /// sizing a production floor from a seam would inflate the budget by whatever a test + /// chose. + /// + /// The PROBE term is the LEGACY-PROBE knob, not `ipfs_max_repos_walked`. Those were + /// one field before the walk cap and the probe budget were split apart, and sizing + /// the probe term off the walk cap would silently read 64 instead of 256. That is a + /// statement about which knob sizes the PROBE term; it is not a claim that the walk + /// cap has no place in the floor, since the walk term above is added to this one + /// rather than substituted for it. /// /// The floor also carries the scan's PAGE toll (#173 round 13, F2): every page the /// legacy scan buys is charged to this same bucket, so a deep scan spends @@ -455,9 +475,13 @@ impl AppState { let pages = config .ipfs_max_legacy_scan_rows .div_ceil(crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS); + let walks = std::cmp::min( + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST as usize, + config.ipfs_max_repos_walked, + ); config .ipfs_rate_limit - .max(config.ipfs_max_legacy_probes + pages) + .max(config.ipfs_max_legacy_probes + pages + walks) } } From a7e8e35030cae4e1126cda6c161bc516e3b8ffff Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:21:44 -0500 Subject: [PATCH 71/77] fix(review): make the coupled sites findable and pin two more properties Code review found the doc comments carrying claims the code does not support and two coupling points documented from one side only. The floor and the resolver's walk_cap are not textual twins: the floor reads the constant, the resolver reads the AppState seam, and they agree only because every construction seeds one from the other. Say that, and give walk_cap the back-reference it lacked. The lifted node_verifies helper no longer named the middleware it mirrors, so an edit to require_signature would not find either copy. The request-target clause pins @path, not @method or content-digest: a 301, 302 or 303 still rewrites a signed POST to a bodyless GET while the signature headers ride along. Record that rather than implying the hop is safe. Add the query-removed and fragment-only matrix rows, and a mutant pinning that scan-phase walks are not charged to the work bucket, which nothing covered. --- crates/git-remote-gitlawb/src/main.rs | 13 ++++++++++--- crates/gitlawb-core/src/redirect.rs | 26 ++++++++++++++++++++++++++ crates/gitlawb-node/src/api/ipfs.rs | 7 +++++++ crates/gitlawb-node/src/state.rs | 15 ++++++++++++--- crates/gl/src/http.rs | 11 ++++++++--- 5 files changed, 63 insertions(+), 9 deletions(-) diff --git a/crates/git-remote-gitlawb/src/main.rs b/crates/git-remote-gitlawb/src/main.rs index 1f779664..738839c1 100644 --- a/crates/git-remote-gitlawb/src/main.rs +++ b/crates/git-remote-gitlawb/src/main.rs @@ -332,8 +332,9 @@ const HTTP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300); /// private repo signs on the retry. reqwest strips only `Authorization`, `Cookie`, /// `Proxy-Authorization` and `WWW-Authenticate` across hosts, so under the default /// `Policy::limited(10)` those signature headers rode a 302 to whatever origin the -/// node named, and on a 307/308 the pack body went with them. Scope the follow to -/// the origin that issued the redirect, which is the same predicate `gl` uses. +/// node named, and on a 307/308 the pack body went with them. Scope the follow to the +/// origin that issued the redirect AND to an identical request-target, which is the +/// same predicate `gl` uses. fn build_http_client() -> Result { Ok(reqwest::blocking::Client::builder() .timeout(HTTP_TIMEOUT) @@ -341,7 +342,8 @@ fn build_http_client() -> Result { .build()?) } -/// Refuse any redirect that leaves the issuing origin, and bound the chain. +/// Refuse any redirect that leaves the issuing origin or rewrites the request-target, +/// and bound the chain. /// /// Refusal is `stop`, not `error`: the 3xx comes back as an ordinary response and /// the caller reports it through the status path it already has. @@ -1140,6 +1142,11 @@ mod tests { /// apart from the mockito seam around them, so an edit to one is visibly an edit to /// both. /// + /// The production verifier both copies mirror is `crate::auth::require_signature` in + /// `crates/gitlawb-node/src/auth/mod.rs`. This is a re-implementation, not a call, so + /// an edit to that middleware has to land here too: otherwise the copies drift and + /// this test keeps passing against a rule the node has stopped applying. + /// /// It asserts internally, which is deliberate but constrains its callers: inside /// `with_body_from_request` those assertions fire on the server thread and reach /// the client as a transport error, not as a recorded verdict. So the identity diff --git a/crates/gitlawb-core/src/redirect.rs b/crates/gitlawb-core/src/redirect.rs index 22830ae7..06460c29 100644 --- a/crates/gitlawb-core/src/redirect.rs +++ b/crates/gitlawb-core/src/redirect.rs @@ -50,6 +50,18 @@ pub const MAX_REDIRECTS: usize = 10; /// signed and unsigned callers alike, for the same reason the predicate is shared: /// two rules would drift. /// +/// The clause pins `@path`, and only `@path`. A gitlawb signature also covers +/// `@method` and `content-digest`, and both of those are still open on a followed +/// hop: on a 301, 302 or 303, reqwest 0.12.28 delegates to tower-http's +/// `FollowRedirect`, which rewrites a POST to a GET and empties the body +/// (tower-http-0.6.8 `src/follow_redirect/mod.rs:273-285`), while its +/// `drop_payload_headers` removes only `Content-Type`, `Content-Length`, +/// `Content-Encoding` and `Transfer-Encoding`. So `Signature`, `Signature-Input` and +/// `Content-Digest` ride along on a request that no longer has the method or the body +/// they were computed over. Only 307 and 308 preserve both. This predicate returning +/// true therefore makes a GET-shaped hop safe to replay against the node's verifier +/// and says nothing about a bodied one: a signed write must not rely on it alone. +/// /// `Url::query` is `None` for `/a` and `Some("")` for `/a?`, and those are two /// different request-targets on the node side too, so the comparison is strict and /// needs no special case. @@ -203,6 +215,20 @@ mod tests { "an empty query added where there was none: a missing query and an empty \ one are different request-targets", ), + ( + "https://node.example/a?x=1", + "https://node.example/a", + false, + "the query dropped where there was one: the comparison is symmetric, and \ + nothing else in the matrix pins that direction", + ), + ( + "https://node.example/a#x", + "https://node.example/a#y", + true, + "a fragment-only difference is still followed: a fragment never reaches \ + the wire, so it is no part of the request-target the node verifies", + ), ]; for (previous, next, expected, why) in cases { assert_eq!( diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 2c078371..a8a37908 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -1516,6 +1516,13 @@ async fn gate_and_serve( // independently of what the source set contains. The taint name stays // "walk-cap": to an operator the meaning is unchanged (a walk ceiling cut // the search), and the knobs still mean what they say, now per phase. + // + // `AppState::ipfs_work_budget` in `crates/gitlawb-node/src/state.rs` + // duplicates this same `min()` as the walk term of the work-bucket floor, + // because a floor that does not reserve what this cap can spend 429s the + // legacy fallback short of its configured reach (#173 round 15, F2). The two + // `min()`s must move together, so an edit starting on this side finds the + // floor rather than only the other way round. let walk_cap = std::cmp::min( state.ipfs_max_history_walks as usize, state.config.ipfs_max_repos_walked, diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 3608089f..3e41d8ec 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -445,9 +445,18 @@ impl AppState { /// it): the provenance phase spends from the budget the floor reserved for the /// search, the fallback 429s short of its configured reach, and the retry re-pays the /// same provenance charges. The term is - /// `min(MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_repos_walked)` because that is - /// textually the resolver's own `walk_cap` in `gate_and_serve`; the two `min()`s must - /// move together, so an edit to either finds the other from here. It reads the + /// `min(MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_repos_walked)` because that is what + /// the resolver's own `walk_cap` in `gate_and_serve` evaluates to. The two + /// expressions are separate, not one shared value: this one reads the CONSTANT + /// `crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST`, while `walk_cap` reads the + /// `AppState` seam `state.ipfs_max_history_walks`. They agree only because every + /// construction seeds that seam from that constant: the production one in `main.rs`, + /// and the two test ones in `auth`'s test module and `test_support`. Nothing + /// mechanically ties the two expressions beyond this comment and its counterpart at + /// `walk_cap` (no shared helper, no type, no assertion outside the one fixture that + /// pins the seam as a precondition), so the two `min()`s have to be moved together + /// by hand, and a construction that seeded the seam from anything else would size + /// the floor for a cap the resolver does not enforce. It reads the /// CONSTANT and the CONFIG knob for the same reason the page term below reads the /// constant page size: `ipfs_max_history_walks` is an `AppState` test seam, and /// sizing a production floor from a seam would inflate the budget by whatever a test diff --git a/crates/gl/src/http.rs b/crates/gl/src/http.rs index 7522c61d..7a28c6f7 100644 --- a/crates/gl/src/http.rs +++ b/crates/gl/src/http.rs @@ -19,10 +19,10 @@ const MAX_ICAPTCHA_RETRIES: usize = 2; /// response body, so it bounds a slow download and not just a slow handshake. const TOTAL_REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); -/// Follow a redirect only when it stays on the origin that issued it, and only for -/// as long as the chain bound allows. +/// Follow a redirect only when it stays on the origin that issued it AND re-issues the +/// identical request-target, and only for as long as the chain bound allows. /// -/// The origin decision itself is [`gitlawb_core::redirect::may_follow`], shared with +/// The decision itself is [`gitlawb_core::redirect::may_follow`], shared with /// `git-remote-gitlawb` so the two signing clients cannot drift apart on it. Refusal /// is `stop`, not `error`: the 3xx comes back as an ordinary response and each caller /// reports it through the status path it already has. @@ -918,6 +918,11 @@ mod tests { /// module). Keep the two textually identical apart from the mockito seam around /// them, so an edit to one is visibly an edit to both. /// + /// The production verifier both copies mirror is `crate::auth::require_signature` in + /// `crates/gitlawb-node/src/auth/mod.rs`. This is a re-implementation, not a call, so + /// an edit to that middleware has to land here too: otherwise the copies drift and + /// this test keeps passing against a rule the node has stopped applying. + /// /// It asserts internally, which is deliberate but constrains its callers: inside /// `with_body_from_request` those assertions fire on the server thread and reach /// the client as a transport error, not as a recorded verdict. So the identity From 69cfbd4b3ccde243ff669392820f0fce86381663 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sat, 15 Aug 2026 18:38:09 -0500 Subject: [PATCH 72/77] fix(node): mint a continuation when a ceiling stops the /ipfs scan mid-page The probe and visit ceilings taint inside gate_and_serve and returned Skip, so the row they refused and every row behind it were walked past without a verdict while the resume position was sealed from pager.cursor, the end of the fetched page. Two ways that stranded content: - On the final page, `pager.exhausted` breaks ahead of every mint arm, so the shed carried no continuation at all. A tokenless search_incomplete is the wrapped-scan answer, which tells `gl ipfs get` its ladder is over, so a holder on that page was unreachable on every retry. - Mid-page, the sealed cursor sat past the refused rows, so the resume skipped them. The shipped ladder tests all set page_rows == ceiling, which puts the break exactly on a page boundary and hides both. The ceiling now returns GateOutcome::CeilingStop rather than tainting on its way out, and the scan loop stops there and seals the row in front of the one that was refused. record_scan_truncation is the single site that records a truncation: it taints and seals together, and only ever moves the position forward, so a later oid candidate re-walking the same rows on a spent budget cannot hand back a token the caller already echoed. The wrapped-scan tail no longer clears a seal, since a ceiling can stop a resumed scan part way through the last page. --- crates/gitlawb-node/src/api/ipfs.rs | 343 ++++++++++++++++++++++++++-- 1 file changed, 328 insertions(+), 15 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index a8a37908..90d19254 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -824,6 +824,14 @@ pub async fn get_by_cid( throttled = true; continue; } + // The provenance path targets a bounded source list rather than the + // table, so there is no scan position to resume from: taint and move on, + // exactly as before. Only the visit ceiling can reach here (the probe + // ceiling is `legacy_scan`-only). + GateOutcome::CeilingStop(reason) => { + record_scan_truncation(&mut walk, &mut scan_continuation, reason, None); + continue; + } GateOutcome::Skip => continue, } } @@ -958,13 +966,21 @@ pub async fn get_by_cid( // byte-identical to the wrapped-scan answer that tells the caller // their ladder is over. if walk.probes >= state.ipfs_max_legacy_probes { - walk.taint("probe-ceiling"); - scan_continuation = pager.cursor.clone(); + record_scan_truncation( + &mut walk, + &mut scan_continuation, + "probe-ceiling", + pager.cursor.clone(), + ); break; } if walk.visits >= state.config.ipfs_max_repo_visits { - walk.taint("visit-ceiling"); - scan_continuation = pager.cursor.clone(); + record_scan_truncation( + &mut walk, + &mut scan_continuation, + "visit-ceiling", + pager.cursor.clone(), + ); break; } // Row ceiling (F2). The two checks above only bind once a probe or a @@ -974,8 +990,12 @@ pub async fn get_by_cid( // probes, anonymously, while holding a scarce walk permit. This is // the check that actually stops that scan. if pager.fetched_rows >= state.ipfs_max_legacy_scan_rows { - walk.taint("row-ceiling"); - scan_continuation = pager.cursor.clone(); + record_scan_truncation( + &mut walk, + &mut scan_continuation, + "row-ceiling", + pager.cursor.clone(), + ); break; } // Rule-bytes ceiling: the row ceiling bounds rows, not the rules each @@ -984,8 +1004,12 @@ pub async fn get_by_cid( // tail is never materialized; `fetch_next_page` drops the rows behind // it and the request that asked for them is the one that truncates. if pager.rule_bytes_exceeded { - walk.taint("rules-ceiling"); - scan_continuation = pager.cursor.clone(); + record_scan_truncation( + &mut walk, + &mut scan_continuation, + "rules-ceiling", + pager.cursor.clone(), + ); break; } // Page toll (F2). Every page is work bought by an anonymous caller, @@ -1038,6 +1062,32 @@ pub async fn get_by_cid( // A throttled walk-requiring candidate is skipped, not fatal: // keep scanning for a later walk-free copy (#173 review, F-C). GateOutcome::Throttled => throttled = true, + // A ceiling refused THIS row, so the resume position is the row in + // front of it, not `pager.cursor`, which by now sits at the end of + // the fetched page and would skip every row the ceiling refused. The + // four arms at the top of the loop seal `pager.cursor` legitimately: + // they only fire once every fetched row has been walked. + // + // Stopping here rather than skipping on is also why the FINAL page is + // covered: the `pager.exhausted` break sits ahead of every mint arm, + // so a ceiling reached while walking the last page used to shed with + // no token at all. + GateOutcome::CeilingStop(reason) => { + // Nothing in front of it means nothing was settled, so there is + // no position to seal and this stop contributes none. The first + // oid candidate cannot get here (the ceiling arms above run + // before every fetch, so a spent budget breaks there instead); + // a LATER candidate can, because it re-walks the already-fetched + // rows from the front without passing those arms. Sealing + // `pager.cursor` for it would push the resume position past rows + // that candidate never examined. + let resume = (idx >= 2).then(|| { + let prev = &pager.rows[idx - 2]; + (prev.created_at_key.clone(), prev.repo.id.clone()) + }); + record_scan_truncation(&mut walk, &mut scan_continuation, reason, resume); + break; + } GateOutcome::Skip => {} } } @@ -1058,9 +1108,13 @@ pub async fn get_by_cid( // // A wrapped scan emits NO continuation: there is nothing left to resume, and the // absence of the token is what tells the caller their ladder is over. - if pager.resumed && pager.exhausted { + // + // Gated on nothing having been sealed: a ceiling can stop a resumed scan PART WAY + // through the last page, which leaves `exhausted` set with rows still unwalked in + // front of the cursor. Clearing the seal there would strand exactly those rows, + // which is the same tokenless dead end this clause exists to describe honestly. + if pager.resumed && pager.exhausted && scan_continuation.is_none() { walk.taint("scan-wrapped"); - scan_continuation = None; } // Nothing served — four distinct tails, in precedence order: @@ -1151,6 +1205,36 @@ enum GateOutcome { /// A walk-requiring candidate hit the per-IP walk quota; skip it but let the caller /// record the throttle so a later walk-free copy can still serve. Throttled, + /// A per-request CEILING (probes, repo visits) refused this row before it could reach + /// a verdict, and will refuse every row after it too. Distinct from `Skip` because the + /// caller must both taint AND seal a resume position in front of this row: the taint + /// alone sheds a 503 whose missing token reads as "ladder over", stranding this row + /// and everything behind it on an inventory that never changes. + CeilingStop(&'static str), +} + +/// The one site that records a scan truncation: taint the walk with the reason and seal +/// the position the caller echoes back, together. +/// +/// Keeping the two together is the point. Every earlier drip on this path was a ceiling +/// that tainted somewhere the mint could not see, so the shed carried no token. +/// +/// The position only ever moves FORWARD. A later oid candidate re-walks the same fetched +/// rows from the front with the request's budget already spent, so it stops earlier than +/// the candidate before it; letting that overwrite the seal would hand back a token the +/// caller already echoed and the ladder would never advance. +fn record_scan_truncation( + walk: &mut WalkState, + slot: &mut Option<(String, String)>, + reason: &'static str, + pos: Option<(String, String)>, +) { + walk.taint(reason); + if let Some(pos) = pos { + if slot.as_ref().is_none_or(|sealed| pos > *sealed) { + *slot = Some(pos); + } + } } /// Outcome of the bounded, off-worker object read for one gated candidate (F6, #173). @@ -1323,9 +1407,10 @@ async fn gate_and_serve( if legacy_scan { if walk.probes >= state.ipfs_max_legacy_probes { // Budget spent: stop probing and mark the scan truncated so the tail - // reports an incomplete search (503), not a false 404 (#173, F2). - walk.taint("probe-ceiling"); - return GateOutcome::Skip; + // reports an incomplete search (503), not a false 404 (#173, F2). The + // CALLER records it, because the resume position belongs to the row this + // refused and only the caller knows it. + return GateOutcome::CeilingStop("probe-ceiling"); } if let Some(key) = crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) @@ -1346,8 +1431,7 @@ async fn gate_and_serve( "/ipfs request hit the per-request repo-visit ceiling \ (GITLAWB_IPFS_MAX_REPO_VISITS); skipping repo without a verdict" ); - walk.taint("visit-ceiling"); - return GateOutcome::Skip; + return GateOutcome::CeilingStop("visit-ceiling"); } walk.visits += 1; @@ -5005,6 +5089,235 @@ mod tests { ); } + /// The probe ceiling must ladder past the row it STOPPED ON, not past the page. + /// + /// The sibling test above sets `ipfs_legacy_scan_page_rows == ipfs_max_legacy_probes`, + /// so the budget runs out exactly at a page boundary and the page-boundary cursor + /// happens to be the right resume point. Misalign the two and it is not: the ceiling + /// taints INSIDE `gate_and_serve`, the loop keeps consuming the rest of the page as + /// `Skip`, and the mint arms at the top of the loop seal `pager.cursor`, which by + /// then sits PAST every row the ceiling refused to probe. Those rows are skipped on + /// the resume as well, and the inventory is stable, so every ladder step reproduces + /// the same gap. + /// + /// Two rows, one probe: the filler spends the budget, the holder is the row the + /// ceiling stops on. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_probe_ceiling_ladders_past_the_row_it_stopped_on(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + // Deliberately NOT equal to the page size: one probe, two rows per page. + state.ipfs_max_legacy_probes = 1; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_readable_repos(&state, "midpage", 1).await; + stamp_scan_order(&pool, "z6readablemidpage/midpage-0000", 0).await; + let (holder_id, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6midpage", + "holder", + b"stopped on this row\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 1).await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.161:5000".parse().unwrap(); + + let mut token: Option = None; + let mut served_at = None; + for step in 1..=4 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + token = Some(continuation_of(&body).unwrap_or_else(|| { + panic!("the probe-ceiling shed at step {step} must carry a continuation: {body}") + })); + } + assert!( + served_at.is_some(), + "the row the probe ceiling stopped on must be reachable on the ladder; \ + a cursor sealed past it strands it on every retry" + ); + } + + /// A ceiling reached on the FINAL page must still mint a continuation. + /// + /// `pager.exhausted` breaks at the top of the loop AHEAD of every mint arm, so a + /// probe or visit ceiling that taints inside `gate_and_serve` while the last page is + /// being walked sheds `search_incomplete` with no token at all. `gl ipfs get` reads a + /// tokenless shed as "the ladder is over" (that is the wrapped-scan contract), so a + /// holder on that page is unreachable, permanently, on an inventory that never + /// changes. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_probe_ceiling_on_the_final_page_still_mints_a_continuation( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // One page holds the whole inventory, so the scan is exhausted the moment it + // starts and the break at `pager.exhausted` is the one that fires. + state.ipfs_legacy_scan_page_rows = 8; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_readable_repos(&state, "finalpage", 1).await; + stamp_scan_order(&pool, "z6readablefinalpage/finalpage-0000", 0).await; + let (holder_id, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6finalpage", + "holder", + b"on the last page\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 1).await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.162:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a probe ceiling on the final page is an incomplete search, not a verdict: {body}" + ); + let token = continuation_of(&body).unwrap_or_else(|| { + panic!( + "a ceiling reached on the final page must still carry a continuation; \ + a tokenless shed tells the caller their ladder is over: {body}" + ) + }); + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "echoing the final-page continuation must reach the holder: {body}" + ); + } + + /// A ceiling on the final page of a RESUMED scan must keep its continuation. + /// + /// `pager.resumed && pager.exhausted` is the wrapped-scan tail: the caller has walked + /// to the end of the table, so there is nothing left to resume and the absent token + /// is the signal. That is only true when the walk actually reached the end. A ceiling + /// stopping part way through the last page leaves rows unwalked in front of the + /// cursor, and clearing the seal there strands them exactly as a tokenless shed does. + /// + /// Four rows, three per page, one probe: the third rung is the one that resumes into + /// a short page and stops on the holder. + /// + /// MUTATION (RED): drop `scan_continuation.is_none()` from the wrap clause. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_ceiling_on_a_resumed_final_page_keeps_its_continuation(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 3; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_readable_repos(&state, "wrapguard", 3).await; + for i in 0..3 { + stamp_scan_order(&pool, &format!("z6readablewrapguard/wrapguard-{i:04}"), i).await; + } + let (holder_id, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6wrapguard", + "holder", + b"behind a resumed ceiling\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 3).await; + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.163:5000".parse().unwrap(); + + let mut token: Option = None; + let mut served_at = None; + for step in 1..=6 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")), + ); + } + assert!( + served_at.is_some(), + "a ceiling that stops part way through the last page of a resumed scan must \ + still ladder; the wrap tail is for a walk that reached the end" + ); + } + /// The VISIT ceiling must advance the ladder too, for the same reason as the probe /// ceiling: it is the sibling arm, it fires on the same root-readable inventory, and /// a tokenless shed there strands everything behind it just as permanently. From 1bdefb18c2bc1e6e3cd370ea61bc7586d09dc76c Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sat, 15 Aug 2026 19:45:08 -0500 Subject: [PATCH 73/77] chore(node): log every /ipfs scan truncation and correct the helper's docs Code review of the previous commit turned up three things worth fixing in place. record_scan_truncation centralized the taint and the seal but logged nothing, so centralizing actually made a truncation less visible than the scattered inline taints it replaced: only the visit ceiling logged, and it logs from inside the gate, before the caller decides whether a position gets sealed. Two identical log lines could therefore mean "the ladder continues" or "the caller is stranded". One debug line now carries the reason and whether a position was sealed. It logs only whether one exists, never its value, since the position names a withheld row's created_at and id. The doc comment claimed to be "the one site that records a scan truncation" while eight other taint sites bypass it. The distinction is real but it is not the one the comment drew: a ceiling stops the scan and owes the caller a position, while a transient skip refuses one row and the rows behind it are still walked. Says that now. The forward-only rule's stated justification was wrong. A later candidate's position is still ahead of the token the caller echoed, so letting it win would not move the ladder backwards; it would shrink each rung toward a single row. Also records that the comparison is Rust byte order while the pager's keyset runs under the database collation, which can disagree on a non-C collation, and why that costs a replay rather than a skipped row. --- crates/gitlawb-node/src/api/ipfs.rs | 43 +++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 90d19254..27bcd13a 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -1207,22 +1207,36 @@ enum GateOutcome { Throttled, /// A per-request CEILING (probes, repo visits) refused this row before it could reach /// a verdict, and will refuse every row after it too. Distinct from `Skip` because the - /// caller must both taint AND seal a resume position in front of this row: the taint - /// alone sheds a 503 whose missing token reads as "ladder over", stranding this row - /// and everything behind it on an inventory that never changes. + /// caller owes the ladder a resume position in front of this row, not just a taint: + /// the taint alone sheds a 503 whose missing token reads as "ladder over", stranding + /// this row and everything behind it on an inventory that never changes. The caller is + /// the only one that can name that position, which is why this returns rather than + /// tainting here. CeilingStop(&'static str), } -/// The one site that records a scan truncation: taint the walk with the reason and seal -/// the position the caller echoes back, together. +/// Taint the walk with a truncation reason and seal the position the caller echoes back, +/// together, at one site. /// -/// Keeping the two together is the point. Every earlier drip on this path was a ceiling -/// that tainted somewhere the mint could not see, so the shed carried no token. +/// Keeping the two together is the point. Every earlier drip on this path was a CEILING +/// that tainted somewhere the mint could not see, so the shed carried no token. This is +/// not the only place the walk is tainted: the transient skips (`acquire`, `read`, +/// `budget`, the walk cap) taint directly and seal nothing, because they refuse one row +/// rather than stopping the scan, and the rows behind them are still walked. A ceiling is +/// what stops the scan, so a ceiling is what owes the caller a position. /// /// The position only ever moves FORWARD. A later oid candidate re-walks the same fetched /// rows from the front with the request's budget already spent, so it stops earlier than -/// the candidate before it; letting that overwrite the seal would hand back a token the -/// caller already echoed and the ladder would never advance. +/// the candidate before it. That earlier position is still ahead of the token the caller +/// echoed, so letting it win would not move the ladder backwards; it would shrink each +/// rung toward a single row and turn a bounded ladder into a crawl. +/// +/// The comparison is Rust's byte-wise `Ord` on `(created_at_key, id)`, while the pager's +/// keyset predicate orders the same TEXT columns under the DATABASE's collation. On a +/// non-`C` collation the two can disagree for ids differing in case or punctuation. The +/// disagreement is one-directional and costs work rather than correctness: the loser is a +/// position behind the true maximum, which REPLAYS rows the caller already walked past +/// instead of skipping rows it never saw. fn record_scan_truncation( walk: &mut WalkState, slot: &mut Option<(String, String)>, @@ -1230,6 +1244,17 @@ fn record_scan_truncation( pos: Option<(String, String)>, ) { walk.taint(reason); + // The one log that separates a rung from a dead end. A truncation that seals nothing + // sheds a tokenless 503, which the client reads as "your ladder is over", so an + // operator staring at a stranded caller needs to see WHICH ceiling stopped the scan + // and whether it handed back a way to continue. The position itself is withheld data + // (its `created_at` and its `id` carry a private repo's owner DID), so log only + // whether one exists, never its value. + tracing::debug!( + reason, + sealed_continuation = pos.is_some(), + "/ipfs legacy scan truncated" + ); if let Some(pos) = pos { if slot.as_ref().is_none_or(|sealed| pos > *sealed) { *slot = Some(pos); From 366b59933112b345113501a65f0f30e76518ff61 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:56:07 -0500 Subject: [PATCH 74/77] fix(node): order the CID candidate list so it cannot depend on heap order oids_for_cid ran a bare SELECT with no ORDER BY, so Postgres was free to return the candidates in physical heap order. get_by_cid walks those candidates under one shared probe budget, visit budget and pager, so whichever comes back first is the one that spends the request's budget: two nodes holding identical data, or one node before and after an unrelated write, could resolve the same CID differently and one could shed a 503 where the other serves. The instability is not hypothetical. An unpin and re-pin of a single object, which is an ordinary production sequence, moves that row to the end of the heap and rotates the list. The sibling pin_sources_for_oid already orders its union for exactly this reason, and the handler comment next to it leans on that determinism. --- crates/gitlawb-node/src/db/mod.rs | 77 +++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index c55906b9..cc2cf0bd 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -2784,11 +2784,19 @@ impl Db { /// candidate lets the handler try each rather than pick one arbitrarily and /// false-404 when the chosen one is withheld or absent while another is /// readable (#173). Empty when the CID was never pinned on this node. + /// + /// ORDERED, for the same reason `pin_sources_for_oid` orders its union: the handler + /// walks these candidates under ONE shared probe budget, visit budget and pager, so + /// whichever comes back first is the one that spends the request's budget. Left + /// unordered this is a bare sequential scan returning heap order, which an unpin and + /// re-pin of any one object rewrites, so two nodes holding identical data could + /// resolve the same CID differently and one could 503 where the other serves. pub async fn oids_for_cid(&self, cid: &str) -> Result> { - let rows = sqlx::query("SELECT sha256_hex FROM pinned_cids WHERE cid = $1") - .bind(cid) - .fetch_all(&self.pool) - .await?; + let rows = + sqlx::query("SELECT sha256_hex FROM pinned_cids WHERE cid = $1 ORDER BY sha256_hex") + .bind(cid) + .fetch_all(&self.pool) + .await?; Ok(rows .into_iter() .map(|r| r.get::("sha256_hex")) @@ -8483,3 +8491,64 @@ mod peers_table_writer_guard { ); } } + +#[cfg(test)] +mod cid_candidate_order_tests { + use super::Db; + use sqlx::PgPool; + + /// The candidate order `oids_for_cid` returns must not depend on the physical + /// row order in `pinned_cids`. + /// + /// `get_by_cid` walks the candidates under ONE shared probe budget, visit budget + /// and pager, so whichever candidate comes back first is the one that spends the + /// request's budget. Without an `ORDER BY` the query is a bare sequential scan and + /// Postgres is free to return heap order, which any UPDATE to any row rewrites: two + /// nodes holding identical data, or one node before and after an unrelated write, + /// resolve the same CID by trying candidates in a different order, so one serves the + /// object and the other sheds a 503. + /// + /// The sibling `pin_sources_for_oid` already orders its union for exactly this + /// reason, and the handler's own comment leans on that determinism. + /// + /// MUTATION (RED): drop the `ORDER BY` and the post-UPDATE read comes back rotated. + #[sqlx::test] + async fn oids_for_cid_is_ordered_independently_of_physical_row_order(pool: PgPool) { + let db = Db::for_testing(pool.clone()); + db.run_migrations().await.unwrap(); + + let cid = "bafkreiorderingfixtureaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + let oids = ["aa".repeat(32), "bb".repeat(32), "cc".repeat(32)]; + for oid in &oids { + db.record_pinned_cid(oid, cid, None).await.unwrap(); + } + + let before = db.oids_for_cid(cid).await.unwrap(); + assert_eq!(before.len(), 3, "fixture must seed three candidates"); + + // Move the first candidate to the end of the heap the way production does it: + // an unpin followed by a re-pin of the same object. An in-place UPDATE is not + // enough, since a HOT update leaves the row reachable from its original item + // pointer and a sequential scan still returns it in its old position. + sqlx::query("DELETE FROM pinned_cids WHERE sha256_hex = $1") + .bind(&oids[0]) + .execute(&pool) + .await + .expect("unpin one candidate"); + db.record_pinned_cid(&oids[0], cid, None).await.unwrap(); + + let after = db.oids_for_cid(cid).await.unwrap(); + assert_eq!( + before, after, + "an unrelated write to one candidate must not reorder the candidate list; \ + the order decides which oid spends the request's shared budget" + ); + + let mut sorted = after.clone(); + sorted.sort(); + assert_eq!( + after, sorted, + "the order must be a stated one (ascending oid), not whatever the heap holds" + ); + } +} From 2656718916ab9c68dd7fccb8f6fba8bba6b43e82 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sat, 15 Aug 2026 22:20:34 -0500 Subject: [PATCH 75/77] feat(core): carry the resumed candidate's identity in the scan token A CID can map to several git oids, and the ladder needs to name which one it is resuming. The sealed position gains the candidate's oid hex so a rung resumes that candidate rather than a position in a list: oids_for_cid is a sorted set, so an ordinal silently repoints at a different candidate when a pin that sorts earlier arrives between rungs, while an identity degrades safely to "not found, restart at the front". The field is length-prefixed and padded to 64, matching the framing the row fields already use, because production oids are 40 hex, not 64: repos are created with --object-format=sha1 and only the test fixtures are sha256. A fixed 64-byte field would fail every seal on a real deployment and shed a tokenless 503, which the client reads as the ladder being over. Both widths are exercised, and a zero-length candidate is rejected at decode so it cannot be confused with the front-of-table sentinel, which is empty row fields with a real candidate. VERSION goes to 3 and the plaintext to 527 bytes, so a token minted under the old layout opens to None and the caller restarts at the front. Nothing has minted one outside tests. The slot carrying the position is a struct rather than a widened tuple on purpose: a 3-tuple would have pulled the hex into the existing keep-the-maximum comparison, changing behavior this commit is meant to leave alone. Token length stays invariant across both oid widths, since length would otherwise be a side channel for the withheld row it names. That is asserted on a real seal in gitlawb-core, not on the gl fixtures: nothing in gl seals or opens a token, so its width constant cannot detect a wrong layout. --- crates/gitlawb-core/src/scan_token.rs | 228 +++++++++++++++++++++++--- crates/gitlawb-node/src/api/ipfs.rs | 62 ++++++- crates/gitlawb-node/src/state.rs | 2 + crates/gl/src/ipfs_cmd.rs | 12 +- 4 files changed, 270 insertions(+), 34 deletions(-) diff --git a/crates/gitlawb-core/src/scan_token.rs b/crates/gitlawb-core/src/scan_token.rs index bc903b1d..f1a6ac81 100644 --- a/crates/gitlawb-core/src/scan_token.rs +++ b/crates/gitlawb-core/src/scan_token.rs @@ -19,11 +19,12 @@ //! position whose plaintext they know XORs two tokens and recovers a withheld //! row's fields in full, strictly worse than emitting plaintext. //! * FIXED-WIDTH plaintext. AEAD ciphertext is plaintext-length plus the tag, and -//! both halves of a scan position vary in length, so a variable encoding would +//! every field of a scan position varies in length, so a variable encoding would //! make token LENGTH a side channel for the sealed row (a short name under a -//! short owner vs a long one). Every token this module mints is byte-identical -//! in length. The two halves are padded to their own separate widths, which is -//! a per-field constant and so still leaks nothing about a given row. +//! short owner vs a long one) and for the candidate oid's width (40 hex on a +//! sha1 repo, 64 on a sha256 one). Every token this module mints is byte-identical +//! in length. Each field is padded to its own separate width, which is a per-field +//! constant and so still leaks nothing about a given row or candidate. //! * The canonical CID as associated data, so a token minted while scanning for //! one CID does not authenticate when replayed against another. //! @@ -45,16 +46,23 @@ pub struct ScanPosition { pub created_at_key: String, /// The row's `id`, the tiebreaking half of the keyset cursor. pub id: String, + /// The oid hex of the CANDIDATE this position resumes. One CID can map to several + /// git oids, so a bare row cursor names a row without naming whose walk it belongs + /// to. The candidate is named by IDENTITY rather than by position in the candidate + /// list: that list is ordered by hex and mutates between rungs, so an index would + /// name a different candidate the moment anything is pinned or unpinned. + pub sha256_hex: String, } /// Plaintext version byte, so a future layout change is a clean open-failure /// (treated as absent) rather than a misparse. /// -/// Bumped to 2 when the two halves stopped sharing one width (see [`ID_WIDTH`]). -/// A token minted under the version-1 layout is a different length and a different +/// Bumped to 2 when the two halves stopped sharing one width (see [`ID_WIDTH`]), and +/// to 3 when the position gained the candidate oid it resumes (see [`OID_WIDTH`]). +/// A token minted under an earlier layout is a different length and a different /// framing, so it opens to `None` and the caller restarts at the front, which is the /// safe direction: a misparse would resume at a fabricated row and skip coverage. -const VERSION: u8 = 2; +const VERSION: u8 = 3; /// Byte width the `created_at` half is padded to. Every value stored here is a /// serialized timestamp, about 30 bytes, so 64 is roomy for the field's whole domain. @@ -79,8 +87,22 @@ const CREATED_WIDTH: usize = 64; /// silently truncating a cursor into one that resumes at the wrong row. const ID_WIDTH: usize = 384; -/// `version | created_len:u16 | created[CREATED_WIDTH] | id_len:u16 | id[ID_WIDTH] | expires:i64` -const PLAINTEXT_LEN: usize = 1 + 2 + CREATED_WIDTH + 2 + ID_WIDTH + 8; +/// Byte width the candidate oid half is padded to. +/// +/// Git mints exactly two oid widths and this field carries BOTH. A production repo is +/// created by `store::init_bare` with `git init --bare --object-format=sha1`, so its +/// oids are 40 hex; only the sha256 test fixtures mint 64. The field is therefore +/// length-prefixed like the two row halves rather than a bare fixed 64: a 64-only +/// field would fail every seal on a real deployment, and a failed seal sheds a +/// tokenless 503 that is byte-identical to "your ladder is over". +/// +/// The padding to 64 is what keeps the WIDTH off the wire. Without it a 40-hex token +/// is 24 bytes shorter than a 64-hex one, and token length would say which object +/// format the holder's repo uses. +const OID_WIDTH: usize = 64; + +/// `version | created_len:u16 | created[CREATED_WIDTH] | id_len:u16 | id[ID_WIDTH] | oid_len:u16 | oid[OID_WIDTH] | expires:i64` +const PLAINTEXT_LEN: usize = 1 + 2 + CREATED_WIDTH + 2 + ID_WIDTH + 2 + OID_WIDTH + 8; /// Nonce width for XChaCha20-Poly1305. const NONCE_LEN: usize = 24; @@ -98,13 +120,14 @@ pub fn new_key() -> [u8; 32] { // broken) without disturbing the other. /// Encode a position into the FIXED-WIDTH plaintext: -/// `version | created_len:u16 | created[CREATED_WIDTH] | id_len:u16 | id[ID_WIDTH] | expires:i64` +/// `version | created_len:u16 | created[CREATED_WIDTH] | id_len:u16 | id[ID_WIDTH] | oid_len:u16 | oid[OID_WIDTH] | expires:i64` /// -/// The padding is the point. AEAD ciphertext is plaintext-length plus the tag, and both -/// halves of a scan position vary in length, so a length-prefixed encoding with no -/// padding would make token LENGTH a side channel for the sealed row. Each half is -/// padded to its OWN fixed width, which keeps every minted token the same length while -/// letting the id half carry the range the write path actually admits. +/// The padding is the point. AEAD ciphertext is plaintext-length plus the tag, and every +/// field of a scan position varies in length, so a length-prefixed encoding with no +/// padding would make token LENGTH a side channel for the sealed row and for the +/// candidate's object format. Each field is padded to its OWN fixed width, which keeps +/// every minted token the same length while letting the id half carry the range the +/// write path actually admits. fn encode_position(pos: &ScanPosition, expires_at_unix: i64) -> anyhow::Result> { let mut out = vec![0u8; PLAINTEXT_LEN]; out[0] = VERSION; @@ -112,6 +135,7 @@ fn encode_position(pos: &ScanPosition, expires_at_unix: i64) -> anyhow::Result width { // Loud rather than truncating: a clipped cursor resumes at the wrong row and @@ -137,8 +161,8 @@ fn decode_position(bytes: &[u8]) -> Option<(ScanPosition, i64)> { return None; } let mut at = 1; - let mut fields = [const { String::new() }; 2]; - for (slot, width) in fields.iter_mut().zip([CREATED_WIDTH, ID_WIDTH]) { + let mut fields = [const { String::new() }; 3]; + for (slot, width) in fields.iter_mut().zip([CREATED_WIDTH, ID_WIDTH, OID_WIDTH]) { let len = u16::from_le_bytes([bytes[at], bytes[at + 1]]) as usize; at += 2; if len > width { @@ -148,8 +172,21 @@ fn decode_position(bytes: &[u8]) -> Option<(ScanPosition, i64)> { at += width; } let expires_at = i64::from_le_bytes(bytes[at..at + 8].try_into().ok()?); - let [created_at_key, id] = fields; - Some((ScanPosition { created_at_key, id }, expires_at)) + let [created_at_key, id, sha256_hex] = fields; + // A zero-length candidate is a third state the encoder never mints. The front-of-table + // sentinel is empty ROW halves with a real oid, so an empty oid would hand the resume + // path a candidate that names nothing; refuse it like every other malformed frame. + if sha256_hex.is_empty() { + return None; + } + Some(( + ScanPosition { + created_at_key, + id, + sha256_hex, + }, + expires_at, + )) } /// AEAD-seal `plaintext` under `key`, bound to `cid`, framed as `nonce || ciphertext`. @@ -202,8 +239,8 @@ fn open_bytes(key: &[u8; 32], cid: &str, raw: &[u8]) -> Option> { /// Seal `pos` under `key`, bound to `cid`, expiring at `expires_at_unix`. /// /// Returns the base64url (no pad) token. Errors only when a field exceeds -/// its half's fixed width ([`CREATED_WIDTH`], [`ID_WIDTH`]) or the AEAD itself fails, -/// never silently truncates. +/// its own fixed width ([`CREATED_WIDTH`], [`ID_WIDTH`], [`OID_WIDTH`]) or the AEAD +/// itself fails, never silently truncates. pub fn seal_scan_token( key: &[u8; 32], cid: &str, @@ -238,13 +275,162 @@ pub fn open_scan_token( mod tests { use super::*; + /// A production-shaped candidate: `git init --bare --object-format=sha1`, so 40 hex. + const OID_40: &str = "a1b2c3d4e5f60718293a4b5c6d7e8f9012345678"; + /// A test-fixture-shaped candidate: the sha256 repos the suite creates mint 64 hex. + const OID_64: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + fn pos(created: &str, id: &str) -> ScanPosition { + pos_for(created, id, OID_40) + } + + fn pos_for(created: &str, id: &str, sha256_hex: &str) -> ScanPosition { ScanPosition { created_at_key: created.to_string(), id: id.to_string(), + sha256_hex: sha256_hex.to_string(), } } + /// Seal a hand-built plaintext through the AEAD half, so a test can frame bytes the + /// encoder would never mint (an old version byte, a zero-length oid) and still exercise + /// the real open path. + fn seal_raw(key: &[u8; 32], cid: &str, plaintext: &[u8]) -> String { + B64URL.encode(seal_bytes(key, cid, plaintext).unwrap()) + } + + /// Byte offset of the oid length prefix inside the plaintext, derived from the widths + /// rather than hardcoded so a width change moves it with the layout. + const OID_LEN_AT: usize = 1 + 2 + CREATED_WIDTH + 2 + ID_WIDTH; + + /// Scenario 1: both of git's oid widths round trip. The 40-hex case is the PRODUCTION + /// shape (`git init --bare --object-format=sha1`), and an all-64 fixture suite would + /// never exercise it, which is exactly how a fixed-64 field would ship broken. + #[test] + fn round_trips_at_both_oid_widths() { + let key = new_key(); + for hex in [OID_40, OID_64] { + let p = pos_for("2020-01-01T00:00:03+00:00", "z6MkOwner/private-repo", hex); + let t = + seal_scan_token(&key, "bafkcid", &p, 1 << 40).expect("both oid widths must seal"); + assert_eq!( + open_scan_token(&key, "bafkcid", &t, 0), + Some(p), + "a {}-hex candidate must open to the identical position", + hex.len() + ); + } + } + + /// Scenario 2: a token framed under the OLD version opens to `None`, never a misparse. + /// Both legs matter: the version-2 plaintext was a different LENGTH, and a future + /// same-length layout would only be caught by the version byte itself. + #[test] + fn a_prior_layout_version_opens_to_none() { + let key = new_key(); + + // The version-2 layout verbatim: no oid field, so 461 bytes. + let mut old = vec![0u8; 1 + 2 + CREATED_WIDTH + 2 + ID_WIDTH + 8]; + old[0] = 2; + assert_eq!(old.len(), 461, "the version-2 plaintext was 461 bytes"); + assert_eq!( + open_scan_token(&key, "bafkcid", &seal_raw(&key, "bafkcid", &old), 0), + None, + "a version-2 token must open to None so the caller restarts at the front" + ); + + // Same length, stale version byte: only the version check can refuse this one. + let mut stamped = encode_position( + &pos("2020-01-01T00:00:03+00:00", "z6MkOwner/private-repo"), + 1 << 40, + ) + .unwrap(); + stamped[0] = VERSION - 1; + assert_eq!( + open_scan_token(&key, "bafkcid", &seal_raw(&key, "bafkcid", &stamped), 0), + None, + "a stale version byte must open to None even at the current width" + ); + } + + /// Scenario 3: token length is invariant across candidate VALUE and candidate WIDTH. + /// The padding is what keeps the oid width off the wire: without it a 40-hex token is + /// 24 bytes shorter than a 64-hex one and the length says which repo format the + /// holder uses. + #[test] + fn token_length_is_invariant_across_oid_widths() { + let key = new_key(); + let created = "2020-01-01T00:00:00+00:00"; + let id = "z6MkOwner/private-repo"; + let short = + seal_scan_token(&key, "bafkcid", &pos_for(created, id, OID_40), 1 << 40).unwrap(); + let long = + seal_scan_token(&key, "bafkcid", &pos_for(created, id, OID_64), 1 << 40).unwrap(); + assert_eq!( + short.len(), + long.len(), + "a 40-hex and a 64-hex candidate must mint tokens of identical length, or the \ + oid width is a side channel" + ); + + // The absolute width, pinned by execution rather than by arithmetic on paper. The + // gl client's mock fixtures hardcode this number (`TOKEN_LEN` in + // crates/gl/src/ipfs_cmd.rs), and nothing in that crate seals a real token, so this + // assertion is the only executable check that the two agree. + assert_eq!( + short.len(), + 756, + "24 nonce + {PLAINTEXT_LEN} plaintext + 16 tag, base64url no pad" + ); + } + + /// Scenario 4: the front-of-table sentinel. Empty row halves with a REAL candidate + /// round trip, which is what lets a seal say "this candidate, no row cursor yet". + #[test] + fn empty_row_fields_round_trip_with_a_real_candidate() { + let key = new_key(); + let p = pos_for("", "", OID_40); + let t = + seal_scan_token(&key, "bafkcid", &p, 1 << 40).expect("the front sentinel must seal"); + assert_eq!(open_scan_token(&key, "bafkcid", &t, 0), Some(p)); + } + + /// Scenario 6, third leg: a zero-length oid is a distinguishable third state that the + /// encoder never mints, and accepting it would hand the sentinel machinery a candidate + /// naming nothing. The decode path refuses it. + #[test] + fn a_zero_length_candidate_opens_to_none() { + let key = new_key(); + let mut plaintext = encode_position( + &pos("2020-01-01T00:00:03+00:00", "z6MkOwner/private-repo"), + 1 << 40, + ) + .unwrap(); + plaintext[OID_LEN_AT..OID_LEN_AT + 2].copy_from_slice(&0u16.to_le_bytes()); + assert_eq!( + open_scan_token(&key, "bafkcid", &seal_raw(&key, "bafkcid", &plaintext), 0), + None, + "a zero-length candidate must open to None, not to a position naming nothing" + ); + } + + /// Scenario 6, second leg: an oid past the 64-byte width fails the seal loudly rather + /// than being clipped into a hex that names a different candidate. + #[test] + fn an_oid_over_the_fixed_width_fails_loudly() { + let key = new_key(); + let p = pos_for( + "2020-01-01T00:00:03+00:00", + "z6MkOwner/private-repo", + &"a".repeat(OID_WIDTH + 1), + ); + assert!( + seal_scan_token(&key, "bafkcid", &p, 1 << 40).is_err(), + "an over-wide candidate must fail the seal, never be truncated into a hex that \ + resumes the wrong candidate" + ); + } + #[test] fn round_trips_under_the_same_key_and_cid() { let key = new_key(); diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 27bcd13a..d6814f04 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -714,7 +714,7 @@ pub async fn get_by_cid( // Set when a ceiling truncates the scan, to the position the caller echoes back. // Sealed at the tail rather than here so exactly one site mints a token and the // wrap case can clear it in one place. - let mut scan_continuation: Option<(String, String)> = None; + let mut scan_continuation: Option = None; for sha256_hex in &oids { // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). @@ -829,7 +829,13 @@ pub async fn get_by_cid( // exactly as before. Only the visit ceiling can reach here (the probe // ceiling is `legacy_scan`-only). GateOutcome::CeilingStop(reason) => { - record_scan_truncation(&mut walk, &mut scan_continuation, reason, None); + record_scan_truncation( + &mut walk, + &mut scan_continuation, + reason, + None, + sha256_hex, + ); continue; } GateOutcome::Skip => continue, @@ -971,6 +977,7 @@ pub async fn get_by_cid( &mut scan_continuation, "probe-ceiling", pager.cursor.clone(), + sha256_hex, ); break; } @@ -980,6 +987,7 @@ pub async fn get_by_cid( &mut scan_continuation, "visit-ceiling", pager.cursor.clone(), + sha256_hex, ); break; } @@ -995,6 +1003,7 @@ pub async fn get_by_cid( &mut scan_continuation, "row-ceiling", pager.cursor.clone(), + sha256_hex, ); break; } @@ -1009,6 +1018,7 @@ pub async fn get_by_cid( &mut scan_continuation, "rules-ceiling", pager.cursor.clone(), + sha256_hex, ); break; } @@ -1085,7 +1095,13 @@ pub async fn get_by_cid( let prev = &pager.rows[idx - 2]; (prev.created_at_key.clone(), prev.repo.id.clone()) }); - record_scan_truncation(&mut walk, &mut scan_continuation, reason, resume); + record_scan_truncation( + &mut walk, + &mut scan_continuation, + reason, + resume, + sha256_hex, + ); break; } GateOutcome::Skip => {} @@ -1162,11 +1178,19 @@ pub async fn get_by_cid( // confidential, not merely tamper-evident (INV-13). A seal failure is not fatal // to the shed: drop the continuation and answer the plain 503, which degrades to // the pre-token behaviour rather than turning a truncation into a 500. - let continuation = scan_continuation.and_then(|(created_at_key, id)| { + let continuation = scan_continuation.and_then(|sealed| { + let SealedScanPos { + row: (created_at_key, id), + sha256_hex, + } = sealed; match gitlawb_core::scan_token::seal_scan_token( &state.ipfs_scan_token_key, &canonical_cid, - &gitlawb_core::scan_token::ScanPosition { created_at_key, id }, + &gitlawb_core::scan_token::ScanPosition { + created_at_key, + id, + sha256_hex, + }, chrono::Utc::now().timestamp() + SCAN_TOKEN_TTL_SECS, ) { Ok(token) => Some(token), @@ -1237,11 +1261,25 @@ enum GateOutcome { /// disagreement is one-directional and costs work rather than correctness: the loser is a /// position behind the true maximum, which REPLAYS rows the caller already walked past /// instead of skipping rows it never saw. +/// A sealed resume position together with the candidate whose walk produced it. +/// +/// The candidate rides along because one CID can map to several git oids, so a bare row +/// pair names a row without naming whose walk it belongs to. It is carried by IDENTITY +/// (the oid hex), never by position in the candidate list, which is ordered by hex and +/// mutates between rungs. +struct SealedScanPos { + /// The keyset row pair, and the only half the forward-only comparison orders on. + row: (String, String), + /// The candidate oid this position resumes. + sha256_hex: String, +} + fn record_scan_truncation( walk: &mut WalkState, - slot: &mut Option<(String, String)>, + slot: &mut Option, reason: &'static str, pos: Option<(String, String)>, + sha256_hex: &str, ) { walk.taint(reason); // The one log that separates a rung from a dead end. A truncation that seals nothing @@ -1256,8 +1294,11 @@ fn record_scan_truncation( "/ipfs legacy scan truncated" ); if let Some(pos) = pos { - if slot.as_ref().is_none_or(|sealed| pos > *sealed) { - *slot = Some(pos); + if slot.as_ref().is_none_or(|sealed| pos > sealed.row) { + *slot = Some(SealedScanPos { + row: pos, + sha256_hex: sha256_hex.to_string(), + }); } } } @@ -6068,6 +6109,7 @@ mod tests { &gitlawb_core::scan_token::ScanPosition { created_at_key: "2020-01-01T00:00:00+00:00".into(), id: "a/b".into(), + sha256_hex: absent_oid(), }, now + 60, ) @@ -6078,6 +6120,7 @@ mod tests { &gitlawb_core::scan_token::ScanPosition { created_at_key: "2020-01-01T00:00:00+00:00".into(), id: format!("did:key:z6MkAVeryLongOwnerKeyIdentifier/{}", "n".repeat(48)), + sha256_hex: absent_oid(), }, now + 60, ) @@ -6157,6 +6200,7 @@ mod tests { &gitlawb_core::scan_token::ScanPosition { created_at_key: scan_order_stamp(3).to_rfc3339(), id: "front-0003".into(), + sha256_hex: absent_oid(), }, now + 3600, ) @@ -6169,6 +6213,7 @@ mod tests { &gitlawb_core::scan_token::ScanPosition { created_at_key: scan_order_stamp(3).to_rfc3339(), id: "front-0003".into(), + sha256_hex: absent_oid(), }, now - 1, ) @@ -6234,6 +6279,7 @@ mod tests { let pos = gitlawb_core::scan_token::ScanPosition { created_at_key: "2020-01-01T00:00:07+00:00".into(), id: "did:key:z6MkHiddenOwner/withheld-repo".into(), + sha256_hex: "f2".repeat(32), }; let expires = chrono::Utc::now().timestamp() + 3600; let first = diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 3e41d8ec..24607e5a 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -1476,6 +1476,8 @@ mod scan_token_key_tests { ScanPosition { created_at_key: "2020-01-01T12:00:00+00:00".to_string(), id: "did:key:z6MkScanTokenOwner/repo".to_string(), + // 40 hex: the production shape, since the node's own repos are sha1. + sha256_hex: "a1b2c3d4e5f60718293a4b5c6d7e8f9012345678".to_string(), } } diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index 3b160a18..93ca5511 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -151,7 +151,7 @@ const MAX_RETRY_AFTER: Duration = Duration::from_secs(5); /// Wait used when a retryable response carries no usable `Retry-After`. const DEFAULT_RETRY_AFTER: Duration = Duration::from_secs(1); -/// Generous ceiling on a continuation token. Real tokens are fixed-width (668 +/// Generous ceiling on a continuation token. Real tokens are fixed-width (756 /// base64url characters today), but the sealed layout has already changed once and /// a rejected token is terminal, so a tight bound would silently kill resume on a /// future version bump. @@ -534,10 +534,12 @@ mod tests { format!("{}{err:#}", diag_text()) } - /// Width of a real continuation today (see the node's scan_token module): 668 + /// Width of a real continuation today (see the node's scan_token module): 756 /// base64url-no-pad characters. The tests build tokens of that width so the - /// fixtures look like the wire, not like a placeholder. - const TOKEN_LEN: usize = 668; + /// fixtures look like the wire, not like a placeholder. Nothing in this crate + /// seals a real token, so the number is pinned on the other side by + /// `token_length_is_invariant_across_oid_widths` in gitlawb-core's scan_token. + const TOKEN_LEN: usize = 756; fn token_of_len(seed: &str, len: usize) -> String { let mut t = String::from(seed); @@ -1199,7 +1201,7 @@ mod tests { "node text must be sanitized before it reaches the terminal, got: {told:?}" ); // The length bound is scoped to the error text, not to `told`: R21 requires a - // stderr line carrying the still-held 668-character token, so no implementation + // stderr line carrying the still-held 756-character token, so no implementation // can keep the whole telling under 600 characters. The error text is where an // uncapped node body would land on this path, so the property still binds. let reported = format!("{err:#}"); From d0635d95b7d601490474419c6bdb99b0ce402e71 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sat, 15 Aug 2026 23:09:36 -0500 Subject: [PATCH 76/77] fix(node): resume the /ipfs scan per candidate so a starved one is not skipped A CID that maps to several oids shared one resume slot across every candidate, and the slot kept the maximum position. An earlier candidate could spend the probe budget walking past a repo that holds the object for a later one, seal a position beyond it, and the next rung would resume past a row that candidate never examined, wrap, and shed tokenless. The client reads an absent token as the ladder being over, so the object became permanently unretrievable, at stock config, deterministically on every retry. The token now names which candidate it is resuming, and the rules that keep that sound are narrower than they first look: Only one candidate per request may seal, and which one depends on where the REQUEST started. On a resumed request it is the resumed candidate alone, since the shared pager holds only the table suffix from the caller's cursor, so a later candidate walked a suffix and never saw the front. On a front-started request it is the first unfinished candidate, since there every candidate walks from the front and a later candidate's stop is honest coverage. Silencing later candidates unconditionally would remove the only thing that mints rung 1 when the first candidate wraps untruncated. A candidate is finished when its row loop walked every fetched row, or when it owed no scan at all. Both matter: a properly provenanced candidate never wraps, so without the second arm the ladder dies every rung. The wrap is witnessed per candidate at the row loop's own two exits, never by reading the shared pager flag at the tail, which any short page sets and which would let a candidate that truncated mid-page look finished and strand the rows it refused. Finishing a non-final candidate advances the seal to the next one at a front sentinel and taints, because the tail emits a continuation only when something tainted; sealing without tainting would suppress the taint and return a definitive 404 while discarding the token it had just minted. The keep-the-maximum comparison is gone. With one proposer per request the slot is written at most once, so an assertion states that directly instead. The pager stays shared per request. A per-candidate pager would restore the fan-out the paging exists to remove. --- crates/gitlawb-node/src/api/ipfs.rs | 1289 ++++++++++++++++++++++++++- 1 file changed, 1241 insertions(+), 48 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index d6814f04..40e58b00 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -700,6 +700,14 @@ pub async fn get_by_cid( // failure class (tampered, a prior boot's key, expired, malformed, minted for a // different CID) lands on the same `None` and starts at the front, silently, // so no probe distinguishes them (INV-13). + // + // The position names the CANDIDATE it resumes as well as the row, so opening it is + // two steps: locate that candidate in the freshly ordered list, then seed the row. + // A sealed hex that is no longer in the list (the object was unpinned under that oid + // between rungs) is treated exactly like an absent token, restarting at the front, + // rather than resumed against some other candidate or turned into a 404 built from a + // table this request never looked at. + let mut resumed_at: Option = None; if let Some(token) = scan_query.scan.as_deref() { if let Some(pos) = gitlawb_core::scan_token::open_scan_token( &state.ipfs_scan_token_key, @@ -707,16 +715,54 @@ pub async fn get_by_cid( token, chrono::Utc::now().timestamp(), ) { - pager.cursor = Some((pos.created_at_key, pos.id)); - pager.resumed = true; + if let Some(at) = oids.iter().position(|oid| *oid == pos.sha256_hex) { + resumed_at = Some(at); + // The empty row pair is the front-of-table sentinel: "this candidate, no + // row cursor yet", which is what the advance to the next candidate seals. + // It cannot collide with a real row: `repos.created_at` is NOT NULL and + // written from a serialized timestamp, and `repos.id` is `{owner}/{name}` + // so it always contains a slash. + pager.cursor = (!pos.created_at_key.is_empty() || !pos.id.is_empty()) + .then_some((pos.created_at_key, pos.id)); + // Set even under the sentinel, where the row walk does start at the front: + // this request SKIPPED the candidates ordered before the resumed one, so + // absence is not proven within it and the tail must keep the retryable + // shed rather than fall through to the definitive 404. + pager.resumed = true; + } } } - // Set when a ceiling truncates the scan, to the position the caller echoes back. - // Sealed at the tail rather than here so exactly one site mints a token and the - // wrap case can clear it in one place. + // The one position the caller echoes back, written at most once per request: by the + // ceiling that truncated the request's proposer, or by that proposer's finish handing + // the ladder to the next oid candidate. Sealed at the tail rather than here so + // exactly one site mints a token and the wrap case can clear it in one place. let mut scan_continuation: Option = None; - - for sha256_hex in &oids { + // True while every candidate ahead of the one being walked FINISHED this request. + // On a front-started request that is the proposer rule: the first candidate that did + // not finish owns the seal, and once it finishes the role passes to the next one. + let mut earlier_all_finished = true; + + for (cand_idx, sha256_hex) in oids.iter().enumerate() { + // Exactly ONE candidate per request may seal a position or advance the ladder, + // and which one depends on where the REQUEST started, not on which candidate is + // interesting. + // + // RESUMED: only the resumed candidate. The pager was seeded from the caller's + // cursor, so `pager.rows` holds the suffix `[start_row, end)`; a later candidate + // that walks "from index 0" walked that suffix and has never seen + // `[front, start_row)`. Letting it seal would record coverage it does not have + // and strand every row in front of the caller's cursor. + // + // FRONT-STARTED: the first candidate that has not finished. Here the suffix + // argument does not exist: every candidate's row loop covers the fetched table + // from the front, so a later candidate's ceiling stop is honest coverage. This + // arm is what mints rung 1 when the first candidate wraps under budget and a + // later one stops on a settled row; silencing it would shed a tainted tokenless + // 503 and end a ladder that works today. + let is_proposer = match resumed_at { + Some(at) => cand_idx == at, + None => earlier_all_finished, + }; // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). // Resolve a PROVENANCED pin by trying each source repo (bounded to // MAX_PIN_SOURCES) through the SAME gate; the first that authorizes serves — no @@ -835,6 +881,7 @@ pub async fn get_by_cid( reason, None, sha256_hex, + is_proposer, ); continue; } @@ -889,9 +936,21 @@ pub async fn get_by_cid( { if state.ipfs_work_rate_limiter.is_throttled(&key).await { throttled = true; + // Skipped for a spent bucket is NOT finished: no scan ran, so nothing was + // covered. Leaving it unfinished keeps the proposer role here, so the + // caller's existing token resumes this candidate once the bucket refills + // instead of the ladder advancing past work that never happened. + earlier_all_finished = false; continue; } } + // Earlier candidates were finished by earlier rungs, so their scans are owed + // nothing. The skip sits HERE on purpose: above it the provenance phase can still + // serve outright from a recorded source, and below it the two marker queries would + // charge a spent-for-nothing lookup pair per skipped candidate. + if resumed_at.is_some_and(|at| cand_idx < at) { + continue; + } let needs_scan = sources.is_empty() || { #[cfg(test)] @@ -934,6 +993,12 @@ pub async fn get_by_cid( } } }; + // Set when THIS candidate's row loop exits having walked every row the pager + // fetched. It is the witness that the candidate covered the table, and it must be + // per candidate: the shared `pager.exhausted` is a per-REQUEST flag set the moment + // any short page is fetched, so a candidate that a ceiling stopped mid-page would + // read as covered and the ladder would advance over the rows it refused. + let mut wrapped = false; if needs_scan { // Walk the candidate repos one bounded page at a time. Pages already // fetched by an earlier oid candidate are re-read from `pager.rows` for @@ -942,6 +1007,7 @@ pub async fn get_by_cid( loop { if idx == pager.rows.len() { if pager.exhausted { + wrapped = true; break; } // Buying another page is only worth its query if a row on it could @@ -978,6 +1044,7 @@ pub async fn get_by_cid( "probe-ceiling", pager.cursor.clone(), sha256_hex, + is_proposer, ); break; } @@ -988,6 +1055,7 @@ pub async fn get_by_cid( "visit-ceiling", pager.cursor.clone(), sha256_hex, + is_proposer, ); break; } @@ -1004,6 +1072,7 @@ pub async fn get_by_cid( "row-ceiling", pager.cursor.clone(), sha256_hex, + is_proposer, ); break; } @@ -1019,6 +1088,7 @@ pub async fn get_by_cid( "rules-ceiling", pager.cursor.clone(), sha256_hex, + is_proposer, ); break; } @@ -1046,6 +1116,14 @@ pub async fn get_by_cid( .fetch_next_page(&state, request_deadline, &cid_str) .await?; if idx == pager.rows.len() { + // The other walked-every-fetched-row exit, and on any inventory + // whose row count is a multiple of the page size it is the NORMAL + // end of the table: `fetch_next_page` only sets `exhausted` on a + // SHORT page, so a full last page leaves the flag clear and the + // empty page after it lands here. Instrumenting only the + // `exhausted` break above leaves `wrapped` false on that path and + // the ladder dies tokenless with later candidates unexamined. + wrapped = true; break; } } @@ -1084,13 +1162,14 @@ pub async fn get_by_cid( // no token at all. GateOutcome::CeilingStop(reason) => { // Nothing in front of it means nothing was settled, so there is - // no position to seal and this stop contributes none. The first - // oid candidate cannot get here (the ceiling arms above run - // before every fetch, so a spent budget breaks there instead); - // a LATER candidate can, because it re-walks the already-fetched - // rows from the front without passing those arms. Sealing - // `pager.cursor` for it would push the resume position past rows - // that candidate never examined. + // no position to seal and this stop contributes none. Only a + // LATER candidate reaches this with nothing settled: it re-walks + // the already-fetched rows from index 0 without passing the + // ceiling arms above, so it can be refused on its very first row. + // The candidate that fetched those rows cannot, because those + // arms run before every fetch and a spent budget breaks there + // instead. Sealing `pager.cursor` here would push the resume + // position past rows this candidate never examined. let resume = (idx >= 2).then(|| { let prev = &pager.rows[idx - 2]; (prev.created_at_key.clone(), prev.repo.id.clone()) @@ -1101,6 +1180,7 @@ pub async fn get_by_cid( reason, resume, sha256_hex, + is_proposer, ); break; } @@ -1108,6 +1188,46 @@ pub async fn get_by_cid( } } } + + // FINISHED: this candidate covered everything it was owed this request, either by + // walking every fetched row (`wrapped`) or by owing no scan at all. The + // `needs_scan` arm is not a nicety: a properly provenanced candidate runs no row + // loop, so without it the resumed candidate can never finish, the advance below + // never fires, and the ladder dies tokenless with later candidates unexamined. + // + // No "and sealed nothing" conjunct: every truncation arm in the row loop breaks + // out of it immediately, so one candidate cannot both seal and walk to the end in + // a single request. + let finished = wrapped || !needs_scan; + if !finished { + earlier_all_finished = false; + } + // The advance. On a RESUMED request the proposer's finish is what moves the ladder + // to the next candidate, sealed at the front-of-table sentinel because that + // candidate has to walk the whole table with a fresh budget. It goes through + // `record_scan_truncation` so the walk is TAINTED as well as sealed: the tail + // emits a continuation only on a tainted walk, so a bare seal here would be + // discarded and the request would fall through to a definitive 404. + // + // Not on a front-started request: there the proposer role simply passes to the + // next unfinished candidate within this same rung, and that candidate seals its + // own stop row. + // + // The final candidate's finish deliberately seals nothing. Absence of a token is + // the ladder's end-of-run signal, and the scan-wrapped clause below turns it into + // the retryable shed. + if is_proposer && finished && resumed_at.is_some() { + if let Some(next) = oids.get(cand_idx + 1) { + record_scan_truncation( + &mut walk, + &mut scan_continuation, + "candidate-advance", + Some((String::new(), String::new())), + next, + true, + ); + } + } } // A RESUMED scan that reached the end of the table has proven absence only over @@ -1123,12 +1243,16 @@ pub async fn get_by_cid( // every other case and turns exactly that incomplete search into a false 404. // // A wrapped scan emits NO continuation: there is nothing left to resume, and the - // absence of the token is what tells the caller their ladder is over. + // absence of the token is what tells the caller their ladder is over. With several + // oid candidates that is the FINAL candidate's wrap; an earlier one's hands the + // ladder on instead, and the seal it leaves in the slot is what keeps this clause off. // - // Gated on nothing having been sealed: a ceiling can stop a resumed scan PART WAY - // through the last page, which leaves `exhausted` set with rows still unwalked in - // front of the cursor. Clearing the seal there would strand exactly those rows, - // which is the same tokenless dead end this clause exists to describe honestly. + // Gated on nothing having been sealed, for two reasons now. A ceiling can stop a + // resumed scan PART WAY through the last page, which leaves `exhausted` set with rows + // still unwalked in front of the cursor; and on a multi-candidate CID the request may + // already carry the advance to the next candidate. Either way the walk is over for + // this rung but the search is not, and clearing the seal would strand exactly what + // the token was minted to reach. if pager.resumed && pager.exhausted && scan_continuation.is_none() { walk.taint("scan-wrapped"); } @@ -1239,28 +1363,6 @@ enum GateOutcome { CeilingStop(&'static str), } -/// Taint the walk with a truncation reason and seal the position the caller echoes back, -/// together, at one site. -/// -/// Keeping the two together is the point. Every earlier drip on this path was a CEILING -/// that tainted somewhere the mint could not see, so the shed carried no token. This is -/// not the only place the walk is tainted: the transient skips (`acquire`, `read`, -/// `budget`, the walk cap) taint directly and seal nothing, because they refuse one row -/// rather than stopping the scan, and the rows behind them are still walked. A ceiling is -/// what stops the scan, so a ceiling is what owes the caller a position. -/// -/// The position only ever moves FORWARD. A later oid candidate re-walks the same fetched -/// rows from the front with the request's budget already spent, so it stops earlier than -/// the candidate before it. That earlier position is still ahead of the token the caller -/// echoed, so letting it win would not move the ladder backwards; it would shrink each -/// rung toward a single row and turn a bounded ladder into a crawl. -/// -/// The comparison is Rust's byte-wise `Ord` on `(created_at_key, id)`, while the pager's -/// keyset predicate orders the same TEXT columns under the DATABASE's collation. On a -/// non-`C` collation the two can disagree for ids differing in case or punctuation. The -/// disagreement is one-directional and costs work rather than correctness: the loser is a -/// position behind the true maximum, which REPLAYS rows the caller already walked past -/// instead of skipping rows it never saw. /// A sealed resume position together with the candidate whose walk produced it. /// /// The candidate rides along because one CID can map to several git oids, so a bare row @@ -1268,20 +1370,48 @@ enum GateOutcome { /// (the oid hex), never by position in the candidate list, which is ordered by hex and /// mutates between rungs. struct SealedScanPos { - /// The keyset row pair, and the only half the forward-only comparison orders on. + /// The keyset row pair to resume that candidate at. The empty pair is the + /// front-of-table sentinel: resume this candidate with no cursor. row: (String, String), /// The candidate oid this position resumes. sha256_hex: String, } +/// Taint the walk with a truncation reason and seal the position the caller echoes back, +/// together, at one site. +/// +/// Keeping the two together is the point. Every earlier drip on this path was a CEILING +/// that tainted somewhere the mint could not see, so the shed carried no token. This is +/// not the only place the walk is tainted: the transient skips (`acquire`, `read`, +/// `budget`, the walk cap) taint directly and seal nothing, because they refuse one row +/// rather than stopping the scan, and the rows behind them are still walked. A ceiling is +/// what stops the scan, so a ceiling is what owes the caller a position. +/// +/// `may_seal` is the caller's proposer verdict, and it is the whole of the multi-candidate +/// rule. Exactly one candidate per request may seal: on a resumed request the resumed +/// candidate (every other one walked only the suffix `[start_row, end)` the pager holds, +/// so its stop is not coverage of the table), and on a front-started request the first +/// candidate that has not finished (there every candidate walks from the front, so a later +/// candidate's stop IS honest coverage). A non-proposer's truncation still TAINTS, since +/// the scan really was cut short, but it contributes no position. +/// +/// That scope, not an ordering comparison, is what keeps the ladder moving forward. Every +/// sealing arm breaks its row loop and only one candidate may seal, so the slot is written +/// at most once per request; the debug assertion below states that invariant where a future +/// change would trip it. The forward-only keep-the-maximum comparison this replaced was +/// the defect: a budget-starved later candidate contributing nothing could not lower a +/// maximum, so the token resumed past rows that candidate had never examined and the CID +/// became permanently unretrievable. fn record_scan_truncation( walk: &mut WalkState, slot: &mut Option, reason: &'static str, pos: Option<(String, String)>, sha256_hex: &str, + may_seal: bool, ) { walk.taint(reason); + let pos = pos.filter(|_| may_seal); // The one log that separates a rung from a dead end. A truncation that seals nothing // sheds a tokenless 503, which the client reads as "your ladder is over", so an // operator staring at a stranded caller needs to see WHICH ceiling stopped the scan @@ -1294,12 +1424,15 @@ fn record_scan_truncation( "/ipfs legacy scan truncated" ); if let Some(pos) = pos { - if slot.as_ref().is_none_or(|sealed| pos > sealed.row) { - *slot = Some(SealedScanPos { - row: pos, - sha256_hex: sha256_hex.to_string(), - }); - } + debug_assert!( + slot.is_none(), + "one candidate per request may seal, and every sealing arm breaks its row \ + loop, so the slot is written at most once" + ); + *slot = Some(SealedScanPos { + row: pos, + sha256_hex: sha256_hex.to_string(), + }); } } @@ -5384,6 +5517,1066 @@ mod tests { ); } + /// A CID with several oid candidates must ladder to a holder only a LATER candidate + /// can serve. + /// + /// `pinned_cids` is unique on the oid, not the cid, so one CID resolves to several + /// candidates and every one of them shares the request's pager, budgets, and resume + /// slot. With a single shared slot the first candidate's truncation seals a row the + /// SECOND candidate never examined: the next rung resumes past the holder, the scan + /// wraps, and the tokenless shed tells the caller the ladder is over. The holder is + /// then unreachable on every retry, because the inventory never changes. + /// + /// Two rows and a two-probe ceiling, with the holder on the second row. The absent + /// candidate sorts first (`oids_for_cid` orders by hex), so it is the one that spends + /// the budget and the holder is reachable only through candidate 2. + /// + /// PRE-FIX (observed RED): rung 1 sheds a token sealing row 1, rung 2 resumes past it, + /// wraps, and sheds with NO token; the holder is never served. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_multi_oid_ladder_reaches_a_holder_only_a_later_candidate_serves( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + // Two probes: exactly the two rows, so candidate 1 spends the whole budget and + // candidate 2 cannot probe anything this rung. + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_readable_repos(&state, "multioid", 1).await; + stamp_scan_order(&pool, "z6readablemultioid/multioid-0000", 0).await; + let (holder_id, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6multioid", + "holder", + b"only the later candidate can serve this\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 1).await; + let cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + + // A second, absent candidate under the SAME cid, sorting ahead of the holder's + // oid so the ordered candidate list puts it first. + let absent_first = "00".repeat(32); + state + .db + .record_pinned_cid(&absent_first, &cid, None) + .await + .expect("co-locate a second source-less oid under the same cid"); + let candidates = state.db.oids_for_cid(&cid).await.unwrap(); + assert_eq!( + candidates, + vec![absent_first.clone(), holder_oid.clone()], + "precondition: the holder's oid must be the SECOND candidate, or the \ + starvation this test is about never happens" + ); + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.164:5000".parse().unwrap(); + + let mut token: Option = None; + let mut served_at = None; + for step in 1..=8 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an intermediate rung is the retryable 503 (step {step}): {body}" + ); + token = Some(continuation_of(&body).unwrap_or_else(|| { + panic!( + "rung {step} shed with no continuation, which tells the caller the \ + ladder is over while a later candidate still holds the object: {body}" + ) + })); + } + assert!( + served_at.is_some(), + "a holder reachable only through a later oid candidate must be served by \ + driving the ladder, not stranded behind the first candidate's seal" + ); + } + + /// Seed `n` root-readable filler rows in scan order, at ascending stamps from + /// `first`. They pass the root gate and hold nothing, so each costs exactly one probe + /// and reaches a clean absent verdict, which is what drives a scan to its probe + /// ceiling on a known row. + async fn seed_ladder_filler( + state: &crate::state::AppState, + pool: &sqlx::PgPool, + prefix: &str, + n: usize, + first: usize, + ) { + seed_root_readable_repos(state, prefix, n).await; + for i in 0..n { + stamp_scan_order( + pool, + &format!("z6readable{prefix}/{prefix}-{i:04}"), + first + i, + ) + .await; + } + } + + /// Open a continuation the node just minted, under the node's own key. The ladder + /// tests that assert WHICH candidate a rung names need the position itself; the status + /// code alone cannot tell "advanced to the next candidate" from "sealed a row of the + /// current one that happens to work". + fn opened(key: &[u8; 32], cid: &str, token: &str) -> gitlawb_core::scan_token::ScanPosition { + gitlawb_core::scan_token::open_scan_token(key, cid, token, chrono::Utc::now().timestamp()) + .expect("the node's own token must open under the node's own key") + } + + /// Mint a continuation the handler will accept, for the fixtures that need to start + /// mid-ladder rather than drive every rung to get there. + fn minted(key: &[u8; 32], cid: &str, sha256_hex: &str, row: (&str, &str)) -> String { + gitlawb_core::scan_token::seal_scan_token( + key, + cid, + &gitlawb_core::scan_token::ScanPosition { + created_at_key: row.0.to_string(), + id: row.1.to_string(), + sha256_hex: sha256_hex.to_string(), + }, + chrono::Utc::now().timestamp() + 300, + ) + .expect("seal a continuation for the fixture") + } + + /// The multi-candidate ladder TERMINATES, and the terminating shed lands exactly on + /// the rung in which the FINAL candidate reaches the end of the table. + /// + /// Every rung must make progress of one of two kinds: advance the row within the + /// resumed candidate, or advance to the next candidate. Neither an endless ladder nor + /// a rung that hands back a token it already issued is acceptable, and a tokenless + /// shed before the last candidate has been walked is the starvation bug wearing the + /// "ladder over" signal. + /// + /// Four rows at two per page against a two-probe ceiling, and two candidates neither + /// of which can serve. The ladder is then fully determined: candidate A takes rungs + /// 1 and 2 on rows (0,1) and (2,3), rung 3 walks A off the end and advances to B, + /// rungs 4 and 5 repeat the table for B, and rung 6 walks B off the end. There are no + /// provenance sources anywhere, so the visit budget is untouched when the scan starts + /// and the settled-no-row shed cannot fire here; the ONLY tokenless rung is the last. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_multi_oid_ladder_ends_when_the_final_candidate_reaches_the_end( + pool: sqlx::PgPool, + ) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "term", 4, 0).await; + let first = "00".repeat(32); + let second = "11".repeat(32); + let cid = seed_legacy_pin(&state, &first).await; + state + .db + .record_pinned_cid(&second, &cid, None) + .await + .expect("co-locate a second source-less oid under the same cid"); + assert_eq!( + state.db.oids_for_cid(&cid).await.unwrap(), + vec![first, second], + "precondition: two candidates in a known order" + ); + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.165:5000".parse().unwrap(); + + let mut token: Option = None; + let mut seen: Vec = Vec::new(); + let mut tokenless_at = None; + for step in 1..=12 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "no candidate can serve, so every rung is the truncated-search 503 \ + (step {step}): {body}" + ); + assert_eq!(body["error"], "search_incomplete", "{body}"); + match continuation_of(&body) { + Some(t) => { + assert!( + !seen.contains(&t), + "rung {step} handed back a token it already issued, which is the \ + ladder spinning in place rather than advancing" + ); + seen.push(t.clone()); + token = Some(t); + } + None => { + tokenless_at = Some(step); + break; + } + } + } + assert_eq!( + tokenless_at, + Some(6), + "the ladder must end on the rung where the SECOND candidate walks off the end \ + of the table: two rungs of rows plus one wrap rung per candidate. An earlier \ + tokenless rung means a candidate was abandoned unexamined" + ); + } + + /// The tokenless shed did not widen: a single-candidate resumed scan that wraps with + /// nothing sealed still ends the ladder exactly as before. + /// + /// This is the negative control for the advance. The advance mints a token whenever a + /// finished candidate has a successor, so an implementation that forgets the successor + /// check would keep minting forever and the caller would never learn the search is + /// over. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_single_candidate_wrap_still_sheds_tokenless(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "solowrap", 2, 0).await; + let cid = seed_legacy_pin(&state, &absent_oid()).await; + assert_eq!( + state.db.oids_for_cid(&cid).await.unwrap().len(), + 1, + "precondition: exactly one candidate, so no advance is ever available" + ); + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.166:5000".parse().unwrap(); + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let token = continuation_of(&body).expect("the probe ceiling mints rung 1"); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a resumed scan that ran off the end has not covered the rows before the \ + token, so it is still the retryable shed: {body}" + ); + assert!( + body["message"] + .as_str() + .is_some_and(|m| m.contains("scan-wrapped")), + "and the reason must still be the wrap, not an advance: {body}" + ); + assert_eq!( + continuation_of(&body), + None, + "with no later candidate the wrap ends the ladder, and the absent token is \ + what tells the caller so: {body}" + ); + } + + /// The advance names the NEXT candidate at the front-of-table sentinel. + /// + /// Asserted on the token's contents rather than on the ladder's outcome, because the + /// outcome alone cannot tell "advanced to candidate B" from "sealed some row of + /// candidate A that happens to work". The sentinel matters on its own: candidate B has + /// walked nothing, so resuming it anywhere but the front skips rows for it. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_finished_candidate_advances_to_the_next_at_the_front_sentinel( + pool: sqlx::PgPool, + ) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "advance", 2, 0).await; + let first = "00".repeat(32); + let second = "11".repeat(32); + let cid = seed_legacy_pin(&state, &first).await; + state + .db + .record_pinned_cid(&second, &cid, None) + .await + .unwrap(); + + let key = state.ipfs_scan_token_key.clone(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.167:5000".parse().unwrap(); + + let (_, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + let rung1 = continuation_of(&body).expect("rung 1 mints on the probe ceiling"); + let pos = gitlawb_core::scan_token::open_scan_token( + &key, + &cid, + &rung1, + chrono::Utc::now().timestamp(), + ) + .expect("the node's own token opens under the node's own key"); + assert_eq!( + pos.sha256_hex, first, + "rung 1 seals the candidate that was actually walking" + ); + assert!( + !pos.created_at_key.is_empty(), + "and it seals a real row, not the sentinel" + ); + + let (_, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&rung1))) + .await + .unwrap(), + ) + .await; + let rung2 = continuation_of(&body) + .expect("the finished candidate must advance the ladder, not end it"); + let pos = gitlawb_core::scan_token::open_scan_token( + &key, + &cid, + &rung2, + chrono::Utc::now().timestamp(), + ) + .expect("the advance token opens"); + assert_eq!( + pos.sha256_hex, second, + "the finished candidate hands the ladder to the NEXT candidate" + ); + assert_eq!( + (pos.created_at_key.as_str(), pos.id.as_str()), + ("", ""), + "at the front-of-table sentinel: the next candidate has walked nothing, so \ + any row cursor would skip rows for it" + ); + } + + /// On a FRONT-STARTED request a later candidate's stop is honest coverage, and it is + /// what mints rung 1 when the first candidate wraps under budget. + /// + /// The rule that silences later candidates is keyed on where the REQUEST started, not + /// on which candidate is walking. On a resumed request the pager holds only the suffix + /// from the caller's cursor, so a later candidate's walk covers a suffix and must not + /// seal. Front-started, the pager starts at the front and every candidate's row loop + /// covers the fetched table from the beginning, so the first candidate that has NOT + /// finished owns the seal, later candidates included. + /// + /// Three rows at three per page against a four-probe ceiling: candidate A walks all + /// three, wraps on the empty page after them, and seals nothing; candidate B spends + /// the fourth probe on row 0 and stops on row 1, with row 0 settled behind it. The + /// holder is row 2, reachable only for B. + /// + /// Over-applying the resumed-only rule here sheds a tainted TOKENLESS 503 at rung 1 + /// and the holder is never served. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_front_started_later_candidate_still_seals_its_stop_row(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 3; + state.ipfs_max_legacy_scan_rows = 1024; + // One more probe than candidate A spends walking the whole table, so A wraps + // UNTRUNCATED and B gets exactly one probe before the ceiling stops it. + state.ipfs_max_legacy_probes = 4; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "frontprop", 2, 0).await; + let (holder_id, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6frontprop", + "holder", + b"reachable only for the later candidate\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 2).await; + let cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + let absent_first = "00".repeat(32); + state + .db + .record_pinned_cid(&absent_first, &cid, None) + .await + .unwrap(); + assert_eq!( + state.db.oids_for_cid(&cid).await.unwrap(), + vec![absent_first, holder_oid.clone()], + "precondition: the holder's oid is the SECOND candidate" + ); + + let key = state.ipfs_scan_token_key.clone(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.168:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let rung1 = continuation_of(&body).unwrap_or_else(|| { + panic!( + "the first candidate wrapped under budget and sealed nothing, so the \ + LATER candidate's ceiling stop is the only thing that can mint rung 1; \ + a tokenless shed here ends a ladder that works today: {body}" + ) + }); + let pos = opened(&key, &cid, &rung1); + assert_eq!( + pos.sha256_hex, holder_oid, + "rung 1 belongs to the candidate that actually stopped" + ); + assert!( + !pos.created_at_key.is_empty(), + "and it seals that candidate's own stop row, not the front sentinel: it \ + walked from the front, so there is nothing to restart" + ); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&rung1))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "echoing rung 1 must reach the holder: {body}" + ); + } + + /// A candidate that a ceiling stopped PART WAY through the last page has not wrapped, + /// however the shared pager's exhausted flag reads. + /// + /// `pager.exhausted` is per REQUEST and is set the moment any short page comes back, + /// so it is true while rows the ceiling refused are still sitting in front of the + /// cursor. Reading it at the tail as the wrap witness marks the truncated candidate + /// finished, advances the ladder to the next one, and strands those rows forever. The + /// witness has to be the per-candidate exit the row loop actually took. + /// + /// Four rows at three per page against a one-probe ceiling. Rung 3 resumes into a + /// SHORT page (two rows), spends its probe on the first and is stopped on the second, + /// so the request ends with `exhausted` set and a row unwalked. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_truncation_on_an_exhausted_page_does_not_advance_the_candidate( + pool: sqlx::PgPool, + ) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 3; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "wrapwitness", 4, 0).await; + let first = "00".repeat(32); + let second = "11".repeat(32); + let cid = seed_legacy_pin(&state, &first).await; + state + .db + .record_pinned_cid(&second, &cid, None) + .await + .unwrap(); + + let key = state.ipfs_scan_token_key.clone(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.169:5000".parse().unwrap(); + + let mut token: Option = None; + for step in 1..=3 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let t = continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")); + let pos = opened(&key, &cid, &t); + assert_eq!( + pos.sha256_hex, first, + "rung {step} stopped the FIRST candidate at a ceiling, so it is still \ + that candidate's rung. Rung 3 is the one that matters: it resumes into \ + a short page, so the shared exhausted flag is set while a row it refused \ + is still unwalked, and an implementation reading that flag as the wrap \ + witness advances here and strands the row" + ); + assert!( + !pos.created_at_key.is_empty(), + "rung {step} seals a real row, not the sentinel" + ); + token = Some(t); + } + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let pos = opened( + &key, + &cid, + &continuation_of(&body).expect("rung 4 walks the first candidate off the end"), + ); + assert_eq!( + (pos.sha256_hex.as_str(), pos.created_at_key.as_str()), + (second.as_str(), ""), + "only once the first candidate has actually walked every fetched row does \ + the ladder advance, and then to the front of the next candidate" + ); + } + + /// A resumed rung does not re-run the scans of candidates earlier rungs already + /// finished, and the skip lands after the provenance phase, not before it. + /// + /// `walk.probes` has no test seam, so the observable is the marker-query pair the + /// fallback gate runs per candidate that reaches `needs_scan`. Both candidates carry + /// recorded sources marked incomplete, so both would bump the counter if both were + /// scanned; resuming at the second must leave it at one. + /// + /// The counter also pins the skip's exact position. Skipping at the top of the oid + /// loop would cut off the provenance phase, which can serve outright; skipping inside + /// `needs_scan` would charge the skipped candidate two lookups for nothing and read 2 + /// here. + #[sqlx::test] + async fn get_by_cid_resumed_rung_skips_the_scans_of_earlier_candidates(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1024; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + // One private repo, used both as the scan inventory and as the recorded pin + // source for each candidate: it denies at the root gate either way, so the + // provenance phase runs and serves nothing. + seed_root_denying_repos(&state, "skipearlier", 2, 0).await; + let source = "skipearlier-0000".to_string(); + let first = "00".repeat(32); + let second = "11".repeat(32); + let cid = seed_legacy_pin(&state, &first).await; + state + .db + .record_pinned_cid(&second, &cid, None) + .await + .unwrap(); + for oid in [&first, &second] { + state.db.record_pin_source(oid, &source).await.unwrap(); + // Incomplete keeps `needs_scan` true past a non-empty source set, which is + // what puts the marker pair on the path for every candidate that is NOT + // skipped. + state.db.mark_pin_sources_incomplete(oid, "").await.unwrap(); + } + + let key = state.ipfs_scan_token_key.clone(); + let token = minted(&key, &cid, &second, ("", "")); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.170:5000".parse().unwrap(); + + crate::api::ipfs::reset_marker_queries(); + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + crate::api::ipfs::marker_queries(), + 1, + "only the resumed candidate owes a scan this rung; the one before it was \ + finished by an earlier rung and must not pay the fallback gate again: {body}" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the resumed candidate walked the table from the sentinel but the rows \ + before the ladder started were skipped, so the honest tail is the retryable \ + shed: {body}" + ); + } + + /// A resumed request still lets LATER candidates serve off the pages it already + /// bought. They are silenced for sealing, not deferred. + /// + /// Skipping them would waste page fetches the caller has already paid for and would + /// turn a rung that could have ended the ladder outright into another 503. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_resumed_rung_still_serves_from_a_later_candidate(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1024; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "opportune", 1, 0).await; + let (holder_id, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6opportune", + "holder", + b"served off a page the resumed candidate bought\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 1).await; + let cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + let absent_first = "00".repeat(32); + state + .db + .record_pinned_cid(&absent_first, &cid, None) + .await + .unwrap(); + + let key = state.ipfs_scan_token_key.clone(); + let token = minted(&key, &cid, &absent_first, ("", "")); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.171:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "a later candidate that can serve off the already-fetched rows must serve in \ + this same rung: {body}" + ); + } + + /// A token naming a candidate that is no longer pinned degrades to a front restart. + /// + /// The hex is sealed by the node so it cannot be forged, but an unpin between rungs + /// can retire it. The open path must then treat the token as absent: never resume + /// some other candidate at that row, never fabricate a 404 out of a table this + /// request has not looked at, and never panic on a lookup that misses. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_token_naming_an_unpinned_candidate_restarts_at_the_front( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1024; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "stalehex", 1, 0).await; + let (holder_id, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6stalehex", + "holder", + b"still reachable after the sealed candidate went away\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 1).await; + let cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + + let key = state.ipfs_scan_token_key.clone(); + // A well-formed token under the node's own key, naming an oid the CID no longer + // resolves to, sealed at a row PAST the holder. Resuming it against the wrong + // candidate would skip the holder; treating it as absent restarts at the front. + let token = minted( + &key, + &cid, + &"cc".repeat(32), + (&scan_order_stamp(9).to_rfc3339(), "zzz/zzz"), + ); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.172:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "a stale candidate identity restarts the scan at the front, so the holder is \ + still found: {body}" + ); + } + + /// The ladder only ever names the resumed candidate, or the one immediately after it. + /// + /// Three candidates, resumed at the first with ceilings that truncate it. Every rung + /// until the first candidate finishes must keep naming it, and the rung that finally + /// moves must hand the ladder to candidate 2 at the front, never skip to candidate 3. + /// Skipping one would mark it finished over a table it never walked. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_ladder_never_skips_a_candidate(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "noskip", 4, 0).await; + let first = "00".repeat(32); + let second = "11".repeat(32); + let third = "22".repeat(32); + let cid = seed_legacy_pin(&state, &first).await; + for oid in [&second, &third] { + state.db.record_pinned_cid(oid, &cid, None).await.unwrap(); + } + assert_eq!( + state.db.oids_for_cid(&cid).await.unwrap(), + vec![first.clone(), second.clone(), third.clone()], + "precondition: three candidates in a known order" + ); + + let key = state.ipfs_scan_token_key.clone(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.173:5000".parse().unwrap(); + + let mut token = Some(minted(&key, &cid, &first, ("", ""))); + let mut moved_to = None; + for step in 1..=8 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let t = continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")); + let pos = opened(&key, &cid, &t); + assert_ne!( + pos.sha256_hex, third, + "rung {step} handed the ladder to the THIRD candidate while the second \ + had not been walked; that marks it finished over a table it never saw" + ); + if pos.sha256_hex != first { + moved_to = Some((pos.sha256_hex.clone(), pos.created_at_key.clone())); + break; + } + token = Some(t); + } + assert_eq!( + moved_to, + Some((second, String::new())), + "the ladder moves one candidate at a time, to the front of the next" + ); + } + + /// R11: a four-candidate CID must be served inside the client's resume budget. + /// + /// `gl ipfs get` stops after `MAX_SCAN_RESUMES` resumes (see + /// `crates/gl/src/ipfs_cmd.rs`; it is private to that crate, so the 8 is repeated + /// here and a change to the cap should bring you to this fixture). Ladder length + /// scales with candidate count, so this is the shape that pins the cost. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_four_candidates_serve_within_the_client_resume_budget(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "fourcand", 1, 0).await; + let (holder_id, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6fourcand", + "holder", + b"four candidates deep\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 1).await; + let cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + // Three absent candidates, all sorting ahead of the holder's oid, so the holder + // is reachable only through the LAST of the four. + for oid in ["00", "11", "22"] { + state + .db + .record_pinned_cid(&oid.repeat(32), &cid, None) + .await + .unwrap(); + } + let candidates = state.db.oids_for_cid(&cid).await.unwrap(); + assert_eq!(candidates.len(), 4, "precondition: four candidates"); + assert_eq!( + candidates[3], holder_oid, + "precondition: the holder's oid sorts last" + ); + + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.174:5000".parse().unwrap(); + + // One initial request plus at most MAX_SCAN_RESUMES echoes, exactly as the client + // drives it. + let mut token: Option = None; + let mut served_at = None; + for step in 1..=9 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + token = Some( + continuation_of(&body) + .unwrap_or_else(|| panic!("rung {step} must carry a continuation: {body}")), + ); + } + assert!( + served_at.is_some_and(|s| s <= 9), + "a four-candidate CID must be served inside the client's 8-resume budget, \ + got {served_at:?}" + ); + } + + /// A resumed candidate that owes NO scan still advances the ladder. + /// + /// Finished means covered, and a candidate whose recorded provenance is complete is + /// covered without a single row being walked. Gating the advance on the row loop + /// having wrapped leaves that candidate permanently unfinished: the rung sheds with + /// no token and every candidate behind it is never examined, which is the starvation + /// bug in a third shape. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_resumed_candidate_owing_no_scan_still_advances(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "noscan", 1, 0).await; + let (holder_id, holder_oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6noscan", + "holder", + b"behind a candidate that owes no scan\n", + ) + .await; + stamp_scan_order(&pool, &holder_id, 1).await; + let cid = seed_legacy_pin_for_oid(&state, &holder_oid).await; + + // The first candidate has a COMPLETE recorded source that denies, so its + // provenance phase answers for it and `needs_scan` is false: no row loop runs and + // it can never wrap. + let absent_first = "00".repeat(32); + state + .db + .record_pinned_cid(&absent_first, &cid, None) + .await + .unwrap(); + seed_root_denying_repos(&state, "noscansrc", 1, 0).await; + state + .db + .record_pin_source(&absent_first, "noscansrc-0000") + .await + .unwrap(); + + let key = state.ipfs_scan_token_key.clone(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.175:5000".parse().unwrap(); + + let mut token = Some(minted(&key, &cid, &absent_first, ("", ""))); + let mut served_at = None; + for step in 1..=6 { + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), token.as_deref())) + .await + .unwrap(), + ) + .await; + if status == StatusCode::OK { + served_at = Some(step); + break; + } + token = Some(continuation_of(&body).unwrap_or_else(|| { + panic!( + "rung {step} shed with no continuation: a candidate that owes no scan \ + is finished, and finished must hand the ladder on: {body}" + ) + })); + } + assert!( + served_at.is_some(), + "the ladder must reach the holder behind the no-scan candidate" + ); + } + + /// Resuming the FINAL candidate at the front sentinel keeps the retryable shed. + /// + /// Under the sentinel the row walk really does start at the front, so it is tempting + /// to treat the request as front-started. It is not: this rung SKIPPED every candidate + /// before the sealed one, so absence has not been proven within it and the definitive + /// 404 is not available. + #[sqlx::test] + async fn get_by_cid_front_sentinel_resume_keeps_the_retryable_shed(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1024; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_denying_repos(&state, "sentinelshed", 2, 0).await; + let first = "00".repeat(32); + let second = "11".repeat(32); + let cid = seed_legacy_pin(&state, &first).await; + state + .db + .record_pinned_cid(&second, &cid, None) + .await + .unwrap(); + + let key = state.ipfs_scan_token_key.clone(); + let token = minted(&key, &cid, &second, ("", "")); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.176:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the candidates before the sealed one were skipped this request, so their \ + absence is unproven and the 404 is not available: {body}" + ); + assert!( + body["message"] + .as_str() + .is_some_and(|m| m.contains("scan-wrapped")), + "{body}" + ); + assert_eq!( + continuation_of(&body), + None, + "the last candidate reached the end of the table, so the ladder is over: {body}" + ); + } + /// The VISIT ceiling must advance the ladder too, for the same reason as the probe /// ceiling: it is the sibling arm, it fires on the same root-readable inventory, and /// a tokenless shed there strands everything behind it just as permanently. From 76f630008f0088c952755b643126affdf962d12b Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sun, 16 Aug 2026 02:20:05 -0500 Subject: [PATCH 77/77] fix(node): never hand back a continuation that made no progress On a resumed request whose visit budget was already spent by the provenance phase, the scan's top-of-loop visit arm sealed pager.cursor, which at that moment is the position the caller just sent. The node returned the caller's own token, verbatim, rung after rung. Three rungs were observed returning an identical position. A token looks like progress, so the client keeps going: gl retries to its resume cap, and every one of those requests re-runs the full provenance phase, up to seventeen repo acquires and cat-file subprocesses, advancing nothing before it errors. That is roughly nine anonymous requests worth of work for none, and it is worse than shedding nothing, because a caller who is told the ladder is over stops immediately. A seal now has to be strictly ahead of where the request itself started: a different candidate is ahead by construction, since only the gated advance can name one, and the same candidate needs a row past the start row. A request that started at the front is before everything, so its seals pass untouched. When the proposer settled at least one row this rung, the existing ceiling arm already seals that row, and it is strictly ahead because a resumed scan only walks rows past its cursor. Only a rung that settled nothing sheds without a token, and that is honest: the spender is the provenance phase, which runs the same way every rung, so no retry can do better. The filter sits at the single mint site, where a future call site cannot bypass it, and it logs the drop as a boolean. record_scan_truncation has already logged that a position was sealed by then, and a 503 carrying no token next to that line is the confusion that log exists to prevent. --- crates/gitlawb-node/src/api/ipfs.rs | 354 +++++++++++++++++++++++++++- 1 file changed, 352 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 40e58b00..92d12980 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -708,6 +708,10 @@ pub async fn get_by_cid( // rather than resumed against some other candidate or turned into a 404 built from a // table this request never looked at. let mut resumed_at: Option = None; + // Where this REQUEST started, kept for the strictly-ahead filter at the mint site. A + // front-started request leaves it `None`, which reads as "before everything", so every + // seal it proposes passes. + let mut scan_start: Option<(String, (String, String))> = None; if let Some(token) = scan_query.scan.as_deref() { if let Some(pos) = gitlawb_core::scan_token::open_scan_token( &state.ipfs_scan_token_key, @@ -717,6 +721,10 @@ pub async fn get_by_cid( ) { if let Some(at) = oids.iter().position(|oid| *oid == pos.sha256_hex) { resumed_at = Some(at); + scan_start = Some(( + pos.sha256_hex.clone(), + (pos.created_at_key.clone(), pos.id.clone()), + )); // The empty row pair is the front-of-table sentinel: "this candidate, no // row cursor yet", which is what the advance to the next candidate seals. // It cannot collide with a real row: `repos.created_at` is NOT NULL and @@ -1302,7 +1310,53 @@ pub async fn get_by_cid( // confidential, not merely tamper-evident (INV-13). A seal failure is not fatal // to the shed: drop the continuation and answer the plain 503, which degrades to // the pre-token behaviour rather than turning a truncation into a 500. - let continuation = scan_continuation.and_then(|sealed| { + // A rung owes a token only when it reached somewhere the caller has not already + // been. `walk.visits` is charged by the provenance phase as well as the scan, so a + // resumed request whose sources spend the ceiling reaches the scan's top-of-loop + // visit arm with nothing fetched, and `pager.cursor` is still the caller's own + // incoming position: sealing it hands them back the token they just sent. `gl` + // echoes a token up to its resume cap, each rung re-running the whole provenance + // phase, so the ladder amplifies one anonymous request into nine while advancing + // nothing and the token makes it look like progress. + // + // Strictly ahead has two arms. A proposal naming the SAME candidate must carry a + // row past the start row. A proposal naming a DIFFERENT candidate is the advance, + // which only a finished candidate can produce, so it is ahead by construction even + // though the front sentinel it seals sorts below every real row. + // + // ONE site, the same argument the single mint site is already built on: a filter + // here cannot be bypassed by a future sealing arm. The ceiling arms stay uniform + // (all of them seal `pager.cursor`) rather than each carrying a copy of this rule. + // + // The row comparison is Rust's byte-wise `Ord` on `(created_at_key, id)`, while the + // pager's keyset predicate ordered the same TEXT columns under the DATABASE's + // collation, so on a non-`C` collation the two can disagree for ids differing in + // case or punctuation. It cannot skip rows: a dropped seal claims no coverage, it + // only ends the rung, and the caller's recovery is a fresh ladder from the front. + // The case this filter exists for is exact equality of a value with itself, which + // no collation moves. + let advancing = scan_continuation.filter(|sealed| { + let advanced = match &scan_start { + None => true, + Some((start_hex, start_row)) => { + sealed.sha256_hex != *start_hex || sealed.row > *start_row + } + }; + if !advanced { + // `record_scan_truncation` already logged `sealed_continuation = true` for + // this seal, and a 503 carrying no token next to that line is exactly the + // confusion that log exists to prevent. This is the correction, and like + // the line it corrects it is a boolean fact only: the position and the + // candidate it names are withheld data. + tracing::debug!( + seal_dropped_not_advancing = true, + "/ipfs dropped a scan continuation that reached no row past the \ + request's own start; shedding without one" + ); + } + advanced + }); + let continuation = advancing.and_then(|sealed| { let SealedScanPos { row: (created_at_key, id), sha256_hex, @@ -1417,7 +1471,9 @@ fn record_scan_truncation( // operator staring at a stranded caller needs to see WHICH ceiling stopped the scan // and whether it handed back a way to continue. The position itself is withheld data // (its `created_at` and its `id` carry a private repo's owner DID), so log only - // whether one exists, never its value. + // whether one exists, never its value. A `true` here is the seal being RECORDED, not + // the response carrying it: the mint site drops a seal that reached no row past the + // request's own start, and logs its own line saying so when it does. tracing::debug!( reason, sealed_continuation = pos.is_some(), @@ -6577,6 +6633,300 @@ mod tests { ); } + /// A rung that advanced nothing must not hand the caller back the token they sent. + /// + /// `walk.visits` is charged by the provenance phase as well as by the scan, so a CID + /// whose recorded sources spend the whole visit budget reaches the scan's top-of-loop + /// visit arm before a single page has been fetched. On a RESUMED request `pager.cursor` + /// is still the caller's own incoming position at that moment, so sealing it emits + /// their own token back verbatim. `gl` echoes a token up to `MAX_SCAN_RESUMES` times + /// inside its deadline, and every one of those rungs re-runs the whole provenance phase + /// (up to `MAX_PIN_SOURCES` acquires and `cat-file` subprocesses) to arrive at the same + /// place, so one anonymous request becomes nine and the token makes the spin look like + /// progress. + /// + /// The STATUS is asserted, not just the missing token. The documented precedence is + /// truncation 503 over throttle 429 over the definitive 404, and a dropped seal must + /// leave the truncation tail standing rather than fall through to either lower one. + /// + /// MUTATION (RED): drop the strictly-ahead filter at the mint site, and the shed + /// carries a continuation that opens to the identical position that was sent. + #[sqlx::test] + async fn get_by_cid_visit_starved_resume_does_not_echo_the_callers_own_token( + pool: sqlx::PgPool, + ) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + // Probes must NOT bind: the visit budget, spent before the scan starts, is what + // stops this request. + state.ipfs_max_legacy_probes = 1024; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repo_visits = 2; + state.config = std::sync::Arc::new(cfg); + + // Four root-readable rows. The first two double as the candidate's recorded pin + // sources: each passes the root gate, so each is charged a visit, and the pair + // spends the ceiling before the scan fetches its first page. + seed_ladder_filler(&state, &pool, "visitstarve", 4, 0).await; + let oid = absent_oid(); + let cid = seed_legacy_pin(&state, &oid).await; + for i in 0..2 { + state + .db + .record_pin_source(&oid, &format!("z6readablevisitstarve/visitstarve-{i:04}")) + .await + .unwrap(); + } + // A non-empty source set only reaches the scan when it may be INCOMPLETE, and the + // scan is what this rung has to be starved out of. + state + .db + .mark_pin_sources_incomplete(&oid, "") + .await + .unwrap(); + + let key = state.ipfs_scan_token_key.clone(); + let start = ( + scan_order_stamp(0).to_rfc3339(), + "z6readablevisitstarve/visitstarve-0000".to_string(), + ); + let token = minted(&key, &cid, &oid, (start.0.as_str(), start.1.as_str())); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.177:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "the search was cut short, so the truncation 503 stands; dropping the seal \ + must not let the tail fall through to the throttle or the definitive 404: \ + {body}" + ); + // Rendered as the position rather than the opaque token so a failure NAMES the + // defect: the echoed triple is byte for byte the one the request carried in. + let echoed = continuation_of(&body).map(|t| { + let pos = opened(&key, &cid, &t); + (pos.sha256_hex, pos.created_at_key, pos.id) + }); + assert_eq!( + echoed, None, + "this rung reached no row the caller had not already been given, so it owes \ + no continuation; echoing {start:?} back under {oid} spins the ladder for \ + another eight amplified requests and calls it progress" + ); + } + + /// The filter drops a seal that stood still, never one that moved. + /// + /// Two properties in one fixture, because they are the two halves of "does not + /// over-drop". Rung 1 is FRONT-STARTED, where the request's start is before every row, + /// so its seal must pass the filter untouched; rung 2 resumes from it, walks two more + /// rows, and its seal must pass because it is strictly ahead. + /// + /// MUTATION (RED): compare the proposal against the start with `>=` instead of `>` + /// and rung 2 keeps its token, so this stays green; compare with `<` and both rungs + /// lose theirs. The filter's job is the middle case, and this fixture is what keeps + /// it from swallowing the other two. + #[sqlx::test] + async fn get_by_cid_a_rung_that_advances_a_row_still_mints(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "advances", 4, 0).await; + let oid = absent_oid(); + let cid = seed_legacy_pin(&state, &oid).await; + + let key = state.ipfs_scan_token_key.clone(); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.178:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .clone() + .oneshot(get_cid_scan(&cid, Some(peer), None)) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let rung1 = continuation_of(&body).expect( + "a front-started request starts before every row, so its probe-ceiling seal \ + is strictly ahead by construction and must still mint", + ); + let first = opened(&key, &cid, &rung1); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&rung1))) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let rung2 = continuation_of(&body) + .expect("the resumed rung walked two more rows, so it has somewhere to seal"); + let second = opened(&key, &cid, &rung2); + assert_eq!( + (second.sha256_hex.as_str(), first.sha256_hex.as_str()), + (oid.as_str(), oid.as_str()), + "one candidate, so both rungs name it" + ); + assert!( + (second.created_at_key.clone(), second.id.clone()) + > (first.created_at_key.clone(), first.id.clone()), + "a rung that reached rows the caller had not seen must seal one of them: \ + {first:?} then {second:?}" + ); + } + + /// The advance to the next candidate is not "backwards", and the filter must know it. + /// + /// The advance seals the front-of-table sentinel, an empty row pair that sorts BELOW + /// every real key. A filter that compared only the row would read the ladder's one real + /// forward step as a step back and drop it, ending every multi-candidate ladder at the + /// rung that was about to hand over. What makes it forward is the candidate: a + /// different hex can only come from the finished-candidate advance, which is ahead by + /// construction. + /// + /// MUTATION (RED): compare rows without first comparing the candidate, and the + /// handover token disappears. + #[sqlx::test] + async fn get_by_cid_the_advance_to_the_next_candidate_survives_the_filter(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.ipfs_legacy_scan_page_rows = 2; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 1024; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_root_denying_repos(&state, "advfilter", 2, 0).await; + let first = "00".repeat(32); + let second = "11".repeat(32); + let cid = seed_legacy_pin(&state, &first).await; + state + .db + .record_pinned_cid(&second, &cid, None) + .await + .unwrap(); + + let key = state.ipfs_scan_token_key.clone(); + // Resumed at the LAST row of the table, so the first candidate's keyset fetch comes + // back empty, it wraps, and the rung's whole job is the handover. + let token = minted( + &key, + &cid, + &first, + (&scan_order_stamp(1).to_rfc3339(), "advfilter-0001"), + ); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.179:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let handover = continuation_of(&body).expect( + "the finished candidate hands the ladder on, and the sentinel it seals is \ + ahead by candidate even though the row pair sorts below the start", + ); + let pos = opened(&key, &cid, &handover); + assert_eq!( + ( + pos.sha256_hex.as_str(), + pos.created_at_key.as_str(), + pos.id.as_str() + ), + (second.as_str(), "", ""), + "the next candidate, at the front of the table" + ); + } + + /// A ceiling that stops mid-page seals the last row it SETTLED, which is progress. + /// + /// This is the arm the tokenless shed must not swallow. A resumed scan only ever walks + /// rows past its start cursor, so any row it settled is strictly ahead, and the rung + /// that settles two rows before a ceiling refuses the third owes the caller the second + /// one. Only a rung that settled NOTHING sheds tokenless, because there the spender is + /// the provenance phase, which runs identically on every retry. + /// + /// MUTATION (RED): seal the request's start instead of the settled row, and the filter + /// (correctly) drops it, so this ladder loses its token and stalls. + #[sqlx::test] + async fn get_by_cid_resumed_ceiling_seals_the_last_row_it_settled(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // A page wide enough that the probe ceiling binds INSIDE the row loop rather than + // at the top of it: that is the arm whose position is the last settled row. + state.ipfs_legacy_scan_page_rows = 4; + state.ipfs_max_legacy_scan_rows = 1024; + state.ipfs_max_legacy_probes = 2; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1024, std::time::Duration::from_secs(3600)); + + seed_ladder_filler(&state, &pool, "settled", 5, 0).await; + let oid = absent_oid(); + let cid = seed_legacy_pin(&state, &oid).await; + + let key = state.ipfs_scan_token_key.clone(); + let start = ( + scan_order_stamp(0).to_rfc3339(), + "z6readablesettled/settled-0000".to_string(), + ); + let token = minted(&key, &cid, &oid, (start.0.as_str(), start.1.as_str())); + let router = ipfs_router(state); + let peer: SocketAddr = "203.0.113.180:5000".parse().unwrap(); + + let (status, body) = status_and_body( + router + .oneshot(get_cid_scan(&cid, Some(peer), Some(&token))) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + let pos = opened( + &key, + &cid, + &continuation_of(&body).expect( + "the rung settled two rows before the ceiling refused the third, so it \ + has real progress to seal", + ), + ); + assert_eq!( + (pos.created_at_key.as_str(), pos.id.as_str()), + ( + scan_order_stamp(2).to_rfc3339().as_str(), + "z6readablesettled/settled-0002" + ), + "the seal is the last row the ceiling let this rung settle, not the row it \ + refused and not the caller's own start" + ); + assert!( + (pos.created_at_key.clone(), pos.id.clone()) > start, + "and it is strictly ahead of the position the request came in with" + ); + } + /// The VISIT ceiling must advance the ladder too, for the same reason as the probe /// ceiling: it is the sibling arm, it fires on the same root-readable inventory, and /// a tokenless shed there strands everything behind it just as permanently.