From 8c697222dcba219cca05d17edf5860a20fe62b39 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Fri, 24 Jul 2026 13:27:28 +0600 Subject: [PATCH 01/26] fix(node): implement reconciliation sweep as durability backstop (#218) Implements periodic reconciliation sweep as durability backstop for dropped replication work (closes #218). Changes: - reconciliation.rs: hourly sweep with cursor-based pagination, per-backend missing-set computation, quarantine rechecks, cooperative shutdown, deadline-bound git scans, stable cursor ordering - db/mod.rs: has_ipfs_cid, filter_ipfs_pinned_oids, filter_pinata_pinned_oids, record_pinned_cid DO UPDATE with WHERE clause, record_pinata_cid NULL cid, migration v12 (DROP NOT NULL), list_all_repos_deduped_stable - ipfs_pin.rs: use has_ipfs_cid instead of is_pinned - main.rs: gate sweep spawn on configured backend - metrics.rs: reconciliation gap counters --- crates/gitlawb-node/src/db/mod.rs | 77 ++++- crates/gitlawb-node/src/ipfs_pin.rs | 7 +- crates/gitlawb-node/src/main.rs | 14 + crates/gitlawb-node/src/metrics.rs | 47 ++- crates/gitlawb-node/src/reconciliation.rs | 382 ++++++++++++++++++++++ 5 files changed, 509 insertions(+), 18 deletions(-) create mode 100644 crates/gitlawb-node/src/reconciliation.rs diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index c6ff644b..edf2de82 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1,8 +1,9 @@ +use std::time::Duration; + use anyhow::{Context, Result}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use sqlx::{postgres::PgPoolOptions, PgPool, Row}; -use std::time::Duration; use tracing::info; use uuid::Uuid; @@ -2260,19 +2261,18 @@ impl Db { // ── Pinned CIDs ─────────────────────────────────────────────────────────────── impl Db { - pub async fn is_pinned(&self, sha256_hex: &str) -> Result { - let row = sqlx::query("SELECT COUNT(*) as cnt FROM pinned_cids WHERE sha256_hex = $1") - .bind(sha256_hex) - .fetch_one(&self.pool) - .await?; - Ok(row.get::("cnt") > 0) - } - + /// Record the local IPFS CID for a git object. + /// If a Pinata-only row already exists (cid IS NULL or was set as Pinata + /// fallback so cid = pinata_cid), this replaces it with the real IPFS CID. pub async fn record_pinned_cid(&self, sha256_hex: &str, cid: &str) -> Result<()> { sqlx::query( "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3) - ON CONFLICT(sha256_hex) DO NOTHING", + ON CONFLICT(sha256_hex) DO UPDATE SET + cid = EXCLUDED.cid, + pinned_at = EXCLUDED.pinned_at + WHERE pinned_cids.cid IS NULL + OR pinned_cids.cid = pinned_cids.pinata_cid", ) .bind(sha256_hex) .bind(cid) @@ -2369,6 +2369,21 @@ impl Db { .collect()) } + /// Returns true when this object has a real local IPFS CID (not a legacy + /// Pinata fallback where cid was set to pinata_cid for new rows). + pub async fn has_ipfs_cid(&self, sha256_hex: &str) -> Result { + let row = sqlx::query( + "SELECT COUNT(*) as cnt FROM pinned_cids + WHERE sha256_hex = $1 + AND cid IS NOT NULL + AND cid IS DISTINCT FROM pinata_cid", + ) + .bind(sha256_hex) + .fetch_one(&self.pool) + .await?; + Ok(row.get::("cnt") > 0) + } + /// Returns true if this object already has a Pinata CID recorded. pub async fn has_pinata_cid(&self, sha256_hex: &str) -> Result { let row = sqlx::query( @@ -2380,9 +2395,45 @@ impl Db { Ok(row.get::("cnt") > 0) } + /// Given a list of sha256_hex values, returns the subset that already have + /// a Pinata CID recorded. Used by the reconciliation sweep to skip objects + /// that Pinata has already handled. + pub async fn filter_pinata_pinned_oids(&self, oids: &[String]) -> Result> { + if oids.is_empty() { + return Ok(Vec::new()); + } + let rows = sqlx::query( + "SELECT sha256_hex FROM pinned_cids WHERE sha256_hex = ANY($1) AND pinata_cid IS NOT NULL", + ) + .bind(oids) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter().map(|r| r.get("sha256_hex")).collect()) + } + + /// Given a list of sha256_hex values, returns the subset that have a real + /// local IPFS CID (excluding legacy Pinata fallback rows where cid equals + /// pinata_cid). Used by the reconciliation sweep to skip IPFS-complete objects. + pub async fn filter_ipfs_pinned_oids(&self, oids: &[String]) -> Result> { + if oids.is_empty() { + return Ok(Vec::new()); + } + let rows = sqlx::query( + "SELECT sha256_hex FROM pinned_cids + WHERE sha256_hex = ANY($1) + AND cid IS NOT NULL + AND cid IS DISTINCT FROM pinata_cid", + ) + .bind(oids) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter().map(|r| r.get("sha256_hex")).collect()) + } + /// Record the Pinata CID for a git object. - /// Inserts the row if it doesn't exist (objects pinned directly to Pinata - /// without a prior local IPFS pin get cid = pinata_cid). + /// `cid` is left NULL for new rows so that `has_ipfs_cid` (which checks + /// `cid IS NOT NULL`) correctly distinguishes local IPFS state from + /// Pinata-only state. pub async fn record_pinata_cid(&self, sha256_hex: &str, pinata_cid: &str) -> Result<()> { sqlx::query( "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) @@ -2390,7 +2441,7 @@ impl Db { ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid", ) .bind(sha256_hex) - .bind(pinata_cid) // fallback local cid if row is new + .bind(Option::<&str>::None) // cid is NULL for Pinata-only new rows .bind(Utc::now().to_rfc3339()) .bind(pinata_cid) .execute(&self.pool) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 3b346190..f48748be 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -107,12 +107,13 @@ pub async fn pin_new_objects( let mut pinned = Vec::new(); for sha in object_list { - // Skip if already pinned - match db.is_pinned(&sha).await { + // Skip if already pinned to local IPFS (checks cid column, + // which is NULL for Pinata-only rows so those will be retried). + match db.has_ipfs_cid(&sha).await { Ok(true) => continue, Ok(false) => {} Err(e) => { - tracing::warn!(sha = %sha, err = %e, "DB error checking pinned status"); + tracing::warn!(sha = %sha, err = %e, "DB error checking IPFS pinned status"); continue; } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index aa0483db..dc372f05 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -16,6 +16,7 @@ mod operator; mod p2p; mod pinata; mod rate_limit; +mod reconciliation; mod server; mod state; mod sync; @@ -498,6 +499,19 @@ async fn main() -> Result<()> { info!("auto-sync worker started"); } + // Periodic reconciliation sweep: re-derives pin/seal sets and fills gaps + // so a dropped replication job never means data loss. + { + let db = state.db.clone(); + let config = Arc::clone(&state.config); + let http_client = Arc::clone(&state.http_client); + let node_keypair = Arc::clone(&state.node_keypair); + let node_did = state.node_did.clone(); + let shutdown_rx = state.subscribe_shutdown(); + reconciliation::spawn(db, config, http_client, node_keypair, node_did, shutdown_rx); + info!("reconciliation sweep worker started"); + } + // On-chain operator setup: verify stake + spawn heartbeat loop if !state.config.contract_node_staking.is_empty() && !state.config.operator_private_key.is_empty() diff --git a/crates/gitlawb-node/src/metrics.rs b/crates/gitlawb-node/src/metrics.rs index 9733129d..bcb13044 100644 --- a/crates/gitlawb-node/src/metrics.rs +++ b/crates/gitlawb-node/src/metrics.rs @@ -15,6 +15,9 @@ //! `gitlawb_pack_size_bytes` //! * a single `gitlawb_info{version, did}` gauge = 1, for joins/dashboards //! * currently-connected peer count — `gitlawb_peers_connected` +//! * reconciliation sweep gaps found and filled — +//! `gitlawb_reconciliation_gaps_found_total` / +//! `gitlawb_reconciliation_gaps_filled_total` //! //! All metrics live in a single process-wide registry initialized by //! [`init`]. Increment helpers (`record_push`, `record_auth_failure`, ...) @@ -33,8 +36,8 @@ use std::sync::OnceLock; use prometheus::{ - Encoder, Histogram, HistogramOpts, IntCounterVec, IntGauge, IntGaugeVec, Opts, Registry, - TextEncoder, + Encoder, Histogram, HistogramOpts, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Opts, + Registry, TextEncoder, }; /// The single, process-wide metrics registry. Initialized by [`init`]. @@ -51,6 +54,8 @@ static SYNC_PROCESSED: OnceLock = OnceLock::new(); static WEBHOOK_DELIVERIES: OnceLock = OnceLock::new(); static PACK_SIZE: OnceLock = OnceLock::new(); static PEERS_CONNECTED: OnceLock = OnceLock::new(); +static RECONCILIATION_GAPS_FOUND: OnceLock = OnceLock::new(); +static RECONCILIATION_GAPS_FILLED: OnceLock = OnceLock::new(); /// One-time initializer. Builds the registry, registers every metric, /// and sets the constant `gitlawb_info` gauge. Idempotent — calling @@ -197,6 +202,30 @@ pub fn init(version: &str, node_did: &str) { .set(peers_connected) .expect("set PEERS_CONNECTED once"); + let gaps_found = IntCounter::with_opts(Opts::new( + "gitlawb_reconciliation_gaps_found_total", + "Total reconciliation sweep gaps detected (objects that should be pinned but are not)", + )) + .expect("gitlawb_reconciliation_gaps_found_total definition"); + registry + .register(Box::new(gaps_found.clone())) + .expect("register gitlawb_reconciliation_gaps_found_total"); + RECONCILIATION_GAPS_FOUND + .set(gaps_found) + .expect("set RECONCILIATION_GAPS_FOUND once"); + + let gaps_filled = IntCounter::with_opts(Opts::new( + "gitlawb_reconciliation_gaps_filled_total", + "Total reconciliation sweep gaps successfully filled (objects pinned by the sweep)", + )) + .expect("gitlawb_reconciliation_gaps_filled_total definition"); + registry + .register(Box::new(gaps_filled.clone())) + .expect("register gitlawb_reconciliation_gaps_filled_total"); + RECONCILIATION_GAPS_FILLED + .set(gaps_filled) + .expect("set RECONCILIATION_GAPS_FILLED once"); + REGISTRY .set(registry) .expect("set REGISTRY once (init must be called exactly once)"); @@ -268,6 +297,20 @@ pub fn set_peers_connected(count: i64) { } } +/// Record reconciliation sweep gaps found (objects that should be pinned but are not). +pub fn record_reconciliation_gaps_found(count: u64) { + if let Some(c) = RECONCILIATION_GAPS_FOUND.get() { + c.inc_by(count); + } +} + +/// Record reconciliation sweep gaps filled (objects successfully pinned by the sweep). +pub fn record_reconciliation_gaps_filled(count: u64) { + if let Some(c) = RECONCILIATION_GAPS_FILLED.get() { + c.inc_by(count); + } +} + /// Encode the registry as the standard Prometheus text exposition format. /// Returns an error if `init` was never called. pub fn encode() -> Result { diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs new file mode 100644 index 00000000..549ce867 --- /dev/null +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -0,0 +1,382 @@ +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::watch; + +use crate::config::Config; +use crate::db::Db; + +/// How often to run a sweep pass. +const SWEEP_INTERVAL_SECS: u64 = 3600; + +/// Maximum repos to process per pass — prevents the sweep from becoming +/// the O(repos) amplification the admission-control work exists to prevent. +const REPOS_PER_PASS: usize = 100; + +/// Maximum objects to pin per backend per repo in a single pass — prevents one +/// large repo from monopolizing the blocking pool or the hourly budget. Applied +/// after filtering out already-pinned objects so the cap reflects actual work. +const MAX_OBJECTS_PER_REPO: usize = 50_000; + +/// Spawn the periodic reconciliation sweep background task. +pub fn spawn( + db: Arc, + config: Arc, + http_client: Arc, + node_keypair: Arc, + node_did: gitlawb_core::did::Did, + mut shutdown_rx: watch::Receiver, +) { + tokio::spawn(async move { + let node_seed = *node_keypair.to_seed(); + let mut cursor = 0usize; + + loop { + let start = std::time::Instant::now(); + match run_pass( + &db, + &config, + &http_client, + &node_seed, + &node_did, + &mut cursor, + &mut shutdown_rx, + ) + .await + { + Ok((count, gaps, filled)) => { + tracing::info!( + repos = count, + gaps_found = gaps, + gaps_filled = filled, + elapsed_ms = start.elapsed().as_millis() as u64, + "reconciliation sweep pass complete" + ); + } + Err(e) => { + tracing::warn!(err = %e, "reconciliation sweep pass failed"); + } + } + + // Check shutdown before sleeping. + if *shutdown_rx.borrow() { + tracing::info!("reconciliation sweep: shutdown signal received, exiting"); + return; + } + + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(SWEEP_INTERVAL_SECS)) => {} + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + tracing::info!("reconciliation sweep: shutdown signal received, exiting"); + return; + } + } + } + } + }); +} + +/// Run one sweep pass. Returns `(repos_scanned, gaps_found, gaps_filled)`. +async fn run_pass( + db: &Db, + config: &Config, + http_client: &reqwest::Client, + node_seed: &[u8; 32], + node_did: &gitlawb_core::did::Did, + cursor: &mut usize, + shutdown_rx: &mut watch::Receiver, +) -> anyhow::Result<(usize, usize, usize)> { + let all = db.list_all_repos_deduped().await?; + + if all.is_empty() { + *cursor = 0; + return Ok((0, 0, 0)); + } + + // Clamp the cursor so a shrinking eligible set never panics. + let start = (*cursor).min(all.len()); + let end = (start + REPOS_PER_PASS).min(all.len()); + let batch = &all[start..end]; + *cursor = if end >= all.len() { 0 } else { end }; + + let mut total_gaps_found = 0usize; + let mut total_gaps_filled = 0usize; + + for repo in batch { + // Cooperative shutdown: exit between repos if signal received. + if *shutdown_rx.borrow() { + tracing::info!("reconciliation sweep: shutdown signal received mid-pass, exiting"); + break; + } + + let repo_slug = format!( + "{}/{}", + crate::db::normalize_owner_key(&repo.owner_did), + repo.name + ); + + let disk = PathBuf::from(&repo.disk_path); + if !disk.exists() { + tracing::warn!(repo = %repo_slug, "disk path missing, skipping"); + continue; + } + + let rules = match db.list_visibility_rules(&repo.id).await { + Ok(r) => r, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "visibility rules fetch failed, skipping"); + continue; + } + }; + + if !crate::visibility::listable_at_root(&rules, repo.is_public, &repo.owner_did, None) { + continue; + } + + let disk_clone = disk.clone(); + let owner_clone = repo.owner_did.clone(); + let rules_clone = rules.clone(); + let is_public = repo.is_public; + let object_list = tokio::task::spawn_blocking(move || -> anyhow::Result> { + let all_objs = crate::git::push_delta::list_all_objects(&disk_clone)?; + let allowed = crate::git::visibility_pack::replicable_blob_set( + &disk_clone, + &rules_clone, + is_public, + &owner_clone, + )?; + let all_blobs = crate::git::push_delta::all_blob_oids(&disk_clone)?; + Ok(crate::git::visibility_pack::replicable_objects_fail_closed( + all_objs, &allowed, &all_blobs, + )) + }) + .await; + + let object_list = match object_list { + Ok(Ok(list)) => list, + Ok(Err(e)) => { + tracing::warn!(repo = %repo_slug, err = %e, "full-scan failed, skipping"); + continue; + } + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "full-scan task panicked, skipping"); + continue; + } + }; + + if object_list.is_empty() { + continue; + } + + // Pre-cap the object list before batch-filtering to keep queries bounded. + let candidates: Vec = if object_list.len() > MAX_OBJECTS_PER_REPO { + tracing::warn!( + repo = %repo_slug, + cap = MAX_OBJECTS_PER_REPO, + total = object_list.len(), + "reconciliation per-repo candidate list truncated to cap" + ); + object_list.into_iter().take(MAX_OBJECTS_PER_REPO).collect() + } else { + object_list + }; + + // ── Phase 1: Public-object pinning (IPFS + Pinata) ──────────────── + // Each backend independently tracks its own completion state, so we + // compute the actually-missing set per backend and cap independently. + + // Recheck quarantine before attempting any external pinning. + match db.is_repo_quarantined(&repo.id).await { + Ok(true) => { + tracing::warn!(repo = %repo_slug, "repo quarantined, skipping public-object pinning"); + // Phase 2 (encrypted) is also skipped — a quarantined repo's + // withheld blobs should not be published either. + continue; + } + Ok(false) => {} + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "quarantine check failed, skipping"); + continue; + } + } + + // Compute IPFS-missing set, capped per-repo. + let already_ipfs = db.filter_ipfs_pinned_oids(&candidates).await?; + let ipfs_missing_set: HashSet<&str> = candidates + .iter() + .map(|s| s.as_str()) + .collect::>() + .difference(&already_ipfs.iter().map(|s| s.as_str()).collect()) + .copied() + .collect(); + let mut ipfs_candidates: Vec = + ipfs_missing_set.into_iter().map(String::from).collect(); + if ipfs_candidates.len() > MAX_OBJECTS_PER_REPO { + ipfs_candidates.truncate(MAX_OBJECTS_PER_REPO); + tracing::warn!( + repo = %repo_slug, + cap = MAX_OBJECTS_PER_REPO, + "IPFS per-repo missing cap reached, truncating" + ); + } + + // Compute Pinata-missing set, capped per-repo. + let already_pinata = db.filter_pinata_pinned_oids(&candidates).await?; + let pinata_missing_set: HashSet<&str> = candidates + .iter() + .map(|s| s.as_str()) + .collect::>() + .difference(&already_pinata.iter().map(|s| s.as_str()).collect()) + .copied() + .collect(); + let mut pinata_candidates: Vec = + pinata_missing_set.into_iter().map(String::from).collect(); + if pinata_candidates.len() > MAX_OBJECTS_PER_REPO { + pinata_candidates.truncate(MAX_OBJECTS_PER_REPO); + tracing::warn!( + repo = %repo_slug, + cap = MAX_OBJECTS_PER_REPO, + "Pinata per-repo missing cap reached, truncating" + ); + } + + let pinned_ipfs = + crate::ipfs_pin::pin_new_objects(&config.ipfs_api, &disk, ipfs_candidates, db).await; + + let pinned_pinata = crate::pinata::pin_new_objects( + http_client, + &config.pinata_upload_url, + &config.pinata_jwt, + &disk, + pinata_candidates, + db, + ) + .await; + + let repo_filled = pinned_ipfs.len() + pinned_pinata.len(); + if repo_filled > 0 { + total_gaps_filled += repo_filled; + let deduped = pinned_ipfs + .iter() + .chain(&pinned_pinata) + .collect::>() + .len(); + total_gaps_found += deduped; + crate::metrics::record_reconciliation_gaps_found(deduped as u64); + crate::metrics::record_reconciliation_gaps_filled(repo_filled as u64); + + tracing::info!( + repo = %repo_slug, + ipfs = pinned_ipfs.len(), + pinata = pinned_pinata.len(), + total = repo_filled, + "reconciliation sweep filled public-object gaps" + ); + } + + // ── Phase 2: Encrypted recovery-copy resealing (withheld blobs) ── + // Only relevant when path-scoped visibility rules exist — without them + // no blobs are withheld and withheld_blob_recipients returns empty. + + // Recheck quarantine before encrypted pinning. + let quarantined = match db.is_repo_quarantined(&repo.id).await { + Ok(q) => q, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "quarantine recheck failed, skipping encrypted pin"); + continue; + } + }; + if quarantined { + tracing::warn!(repo = %repo_slug, "repo quarantined, skipping encrypted pinning"); + continue; + } + + let has_path_scoped = crate::git::visibility_pack::has_path_scoped_rule(&rules); + if has_path_scoped && !config.ipfs_api.is_empty() { + let p = disk.clone(); + let owner = repo.owner_did.clone(); + let r = rules.clone(); + let is_public_2 = repo.is_public; + let recipients = tokio::task::spawn_blocking(move || { + crate::git::visibility_pack::withheld_blob_recipients(&p, &r, is_public_2, &owner) + }) + .await; + + match recipients { + Ok(Ok(rec)) if !rec.is_empty() => { + let sealed = crate::encrypted_pin::encrypt_and_pin( + &config.ipfs_api, + &disk, + db, + &repo.id, + node_seed, + &rec, + ) + .await; + + // Anchor ALL existing encrypted blobs for this repo, not + // just the ones encrypted this pass. This ensures that if + // a prior manifest anchor failed the retry will include + // previously-encrypted blobs too. + let all_existing = db.list_all_encrypted_blobs(&repo.id).await?; + if !all_existing.is_empty() && !config.irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&repo.owner_did); + let slug = format!("{}/{}", owner_short, repo.name); + let ts = chrono::Utc::now().to_rfc3339(); + let node_did_str = node_did.to_string(); + + // Merge existing blobs with freshly-sealed ones, + // preferring later entries (newly-sealed) on conflict. + let mut blob_map: HashMap = HashMap::new(); + for (oid, cid) in &all_existing { + blob_map.insert(oid.clone(), cid.clone()); + } + for (oid, cid) in &sealed { + blob_map.insert(oid.clone(), cid.clone()); + } + let merged: Vec<(String, String)> = blob_map.into_iter().collect(); + + let manifest = crate::arweave::EncryptedManifest { + repo: &slug, + owner_did: &repo.owner_did, + node_did: &node_did_str, + timestamp: &ts, + blobs: &merged, + }; + if let Err(e) = crate::arweave::anchor_encrypted_manifest( + http_client, + &config.irys_url, + &manifest, + ) + .await + { + tracing::warn!( + repo = %slug, + err = %e, + "encrypted manifest anchor failed (will retry next pass)" + ); + } + } + } + Ok(Ok(_)) => {} + Ok(Err(e)) => { + tracing::warn!( + repo = %repo_slug, + err = %e, + "withheld_blob_recipients failed, skipping encrypted pin" + ); + } + Err(e) => { + tracing::warn!( + repo = %repo_slug, + err = %e, + "withheld_blob_recipients task panicked, skipping encrypted pin" + ); + } + } + } + } + + Ok((batch.len(), total_gaps_found, total_gaps_filled)) +} From b40702a9e4abebea4f08f41a2bd2c57ba5e6a8b7 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Fri, 24 Jul 2026 13:40:32 +0600 Subject: [PATCH 02/26] fix(reconciliation): address review findings - [P1] Migration v12: DROP NOT NULL on pinned_cids.cid - [P2] Subtract already-pinned before per-repo cap (no pre-cap) - [P2] Stable sweep cursor via list_all_repos_deduped_stable - [P2] Deadlines on blocking git scans (REPO_SCAN_DEADLINE) - [P2] Gate manifest anchor on newly-sealed content - [P3] Gate sweep spawn on configured backend - [P3] DB errors skip repo, not abort pass --- crates/gitlawb-node/src/db/mod.rs | 33 +++ crates/gitlawb-node/src/reconciliation.rs | 299 +++++++++++++--------- 2 files changed, 204 insertions(+), 128 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index edf2de82..42b86881 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -884,6 +884,18 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE received_ref_updates ADD COLUMN IF NOT EXISTS owner_did TEXT", ], }, + Migration { + version: 12, + name: "pinned_cids_cid_nullable", + stmts: &[ + // Allow cid to be NULL so record_pinata_cid can create Pinata-only + // rows without a local IPFS CID. has_ipfs_cid uses + // "cid IS NOT NULL AND cid IS DISTINCT FROM pinata_cid" so that + // legacy rows where cid was set to pinata_cid as fallback are + // still correctly classified. + "ALTER TABLE pinned_cids ALTER COLUMN cid DROP NOT NULL", + ], + }, // Reservation: v17, deliberately not main's current_max + 1 (which is 12). // The runner keys the applied set on the integer alone, so a version another // in-flight branch also claims is skipped in full on whichever side merges @@ -1250,6 +1262,27 @@ impl Db { Ok(rows.into_iter().map(row_to_repo).collect()) } + /// Like `list_all_repos_deduped` but ordered by a stable key (`id`) so a + /// positional cursor deterministically covers every repo regardless of push + /// activity. Used by the reconciliation sweep to avoid starving idle repos. + pub async fn list_all_repos_deduped_stable(&self) -> Result> { + let sql = format!( + "{} + SELECT d.id, d.name, d.owner_did, d.description, d.is_public, + d.default_branch, d.created_at, d.updated_at, d.disk_path, + d.forked_from, d.machine_id + FROM deduped d + ORDER BY d.id ASC", + Self::dedup_cte() + ); + let rows = sqlx::query(&sql) + .bind(None::<&str>) + .fetch_all(&self.pool) + .await?; + + Ok(rows.into_iter().map(row_to_repo).collect()) + } + /// Repos currently quarantined (admitted as mirrors but withheld from every /// listing surface). `list_all_repos_deduped` excludes these (its `DEDUP_CTE` /// filters `quarantined = FALSE`), so a gate that resolves a slug against the diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs index 549ce867..cd700b6f 100644 --- a/crates/gitlawb-node/src/reconciliation.rs +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -1,6 +1,7 @@ use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use std::sync::Arc; +use std::time::Duration; use tokio::sync::watch; use crate::config::Config; @@ -18,7 +19,12 @@ const REPOS_PER_PASS: usize = 100; /// after filtering out already-pinned objects so the cap reflects actual work. const MAX_OBJECTS_PER_REPO: usize = 50_000; +/// Per-repo deadline for the blocking git scan (list_all_objects + visibility +/// filter). A pathological repo that stalls past this is skipped for the pass. +const REPO_SCAN_DEADLINE: Duration = Duration::from_secs(300); + /// Spawn the periodic reconciliation sweep background task. +/// No-op when neither IPFS nor Pinata is configured. pub fn spawn( db: Arc, config: Arc, @@ -27,6 +33,11 @@ pub fn spawn( node_did: gitlawb_core::did::Did, mut shutdown_rx: watch::Receiver, ) { + if config.ipfs_api.is_empty() && config.pinata_jwt.is_empty() { + tracing::info!("reconciliation sweep: neither IPFS nor Pinata configured, skipping spawn"); + return; + } + tokio::spawn(async move { let node_seed = *node_keypair.to_seed(); let mut cursor = 0usize; @@ -58,7 +69,6 @@ pub fn spawn( } } - // Check shutdown before sleeping. if *shutdown_rx.borrow() { tracing::info!("reconciliation sweep: shutdown signal received, exiting"); return; @@ -87,14 +97,15 @@ async fn run_pass( cursor: &mut usize, shutdown_rx: &mut watch::Receiver, ) -> anyhow::Result<(usize, usize, usize)> { - let all = db.list_all_repos_deduped().await?; + // Use stable ordering so a positional cursor deterministically covers every + // repo regardless of push activity — idle repos are not starved. + let all = db.list_all_repos_deduped_stable().await?; if all.is_empty() { *cursor = 0; return Ok((0, 0, 0)); } - // Clamp the cursor so a shrinking eligible set never panics. let start = (*cursor).min(all.len()); let end = (start + REPOS_PER_PASS).min(all.len()); let batch = &all[start..end]; @@ -104,7 +115,6 @@ async fn run_pass( let mut total_gaps_filled = 0usize; for repo in batch { - // Cooperative shutdown: exit between repos if signal received. if *shutdown_rx.borrow() { tracing::info!("reconciliation sweep: shutdown signal received mid-pass, exiting"); break; @@ -134,64 +144,58 @@ async fn run_pass( continue; } + // Bound the blocking git scan with a deadline so a pathological repo + // cannot stall the entire pass. let disk_clone = disk.clone(); let owner_clone = repo.owner_did.clone(); let rules_clone = rules.clone(); let is_public = repo.is_public; - let object_list = tokio::task::spawn_blocking(move || -> anyhow::Result> { - let all_objs = crate::git::push_delta::list_all_objects(&disk_clone)?; - let allowed = crate::git::visibility_pack::replicable_blob_set( - &disk_clone, - &rules_clone, - is_public, - &owner_clone, - )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&disk_clone)?; - Ok(crate::git::visibility_pack::replicable_objects_fail_closed( - all_objs, &allowed, &all_blobs, - )) - }) + let object_list = tokio::time::timeout( + REPO_SCAN_DEADLINE, + tokio::task::spawn_blocking(move || -> anyhow::Result> { + let all_objs = crate::git::push_delta::list_all_objects(&disk_clone)?; + let allowed = crate::git::visibility_pack::replicable_blob_set( + &disk_clone, + &rules_clone, + is_public, + &owner_clone, + )?; + let all_blobs = crate::git::push_delta::all_blob_oids(&disk_clone)?; + Ok(crate::git::visibility_pack::replicable_objects_fail_closed( + all_objs, &allowed, &all_blobs, + )) + }), + ) .await; - let object_list = match object_list { - Ok(Ok(list)) => list, - Ok(Err(e)) => { + let object_list: Vec = match object_list { + Ok(Ok(Ok(list))) => list, + Ok(Ok(Err(e))) => { tracing::warn!(repo = %repo_slug, err = %e, "full-scan failed, skipping"); continue; } - Err(e) => { + Ok(Err(e)) => { tracing::warn!(repo = %repo_slug, err = %e, "full-scan task panicked, skipping"); continue; } + Err(_) => { + tracing::warn!(repo = %repo_slug, "full-scan deadline exceeded, skipping"); + continue; + } }; if object_list.is_empty() { continue; } - // Pre-cap the object list before batch-filtering to keep queries bounded. - let candidates: Vec = if object_list.len() > MAX_OBJECTS_PER_REPO { - tracing::warn!( - repo = %repo_slug, - cap = MAX_OBJECTS_PER_REPO, - total = object_list.len(), - "reconciliation per-repo candidate list truncated to cap" - ); - object_list.into_iter().take(MAX_OBJECTS_PER_REPO).collect() - } else { - object_list - }; - // ── Phase 1: Public-object pinning (IPFS + Pinata) ──────────────── - // Each backend independently tracks its own completion state, so we - // compute the actually-missing set per backend and cap independently. - + // Compute the actually-missing set per backend from the FULL object + // list (no pre-cap) so trailing objects are never excluded. The cap + // applies to the missing sets, bounding pin work. // Recheck quarantine before attempting any external pinning. match db.is_repo_quarantined(&repo.id).await { Ok(true) => { - tracing::warn!(repo = %repo_slug, "repo quarantined, skipping public-object pinning"); - // Phase 2 (encrypted) is also skipped — a quarantined repo's - // withheld blobs should not be published either. + tracing::warn!(repo = %repo_slug, "repo quarantined, skipping"); continue; } Ok(false) => {} @@ -201,55 +205,67 @@ async fn run_pass( } } - // Compute IPFS-missing set, capped per-repo. - let already_ipfs = db.filter_ipfs_pinned_oids(&candidates).await?; - let ipfs_missing_set: HashSet<&str> = candidates - .iter() - .map(|s| s.as_str()) - .collect::>() - .difference(&already_ipfs.iter().map(|s| s.as_str()).collect()) - .copied() - .collect(); - let mut ipfs_candidates: Vec = - ipfs_missing_set.into_iter().map(String::from).collect(); - if ipfs_candidates.len() > MAX_OBJECTS_PER_REPO { - ipfs_candidates.truncate(MAX_OBJECTS_PER_REPO); - tracing::warn!( - repo = %repo_slug, - cap = MAX_OBJECTS_PER_REPO, - "IPFS per-repo missing cap reached, truncating" - ); - } + // IPFS-missing set (capped). + let already_ipfs = match db.filter_ipfs_pinned_oids(&object_list).await { + Ok(v) => v, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "filter_ipfs_pinned_oids failed, skipping"); + continue; + } + }; + let ipfs_missing: Vec = { + let all_set: HashSet<&str> = object_list.iter().map(|s| s.as_str()).collect(); + let done_set: HashSet<&str> = already_ipfs.iter().map(|s| s.as_str()).collect(); + let mut v: Vec = all_set + .difference(&done_set) + .map(|s| s.to_string()) + .collect(); + if v.len() > MAX_OBJECTS_PER_REPO { + v.truncate(MAX_OBJECTS_PER_REPO); + tracing::warn!( + repo = %repo_slug, + cap = MAX_OBJECTS_PER_REPO, + "IPFS per-repo missing cap reached, truncating" + ); + } + v + }; - // Compute Pinata-missing set, capped per-repo. - let already_pinata = db.filter_pinata_pinned_oids(&candidates).await?; - let pinata_missing_set: HashSet<&str> = candidates - .iter() - .map(|s| s.as_str()) - .collect::>() - .difference(&already_pinata.iter().map(|s| s.as_str()).collect()) - .copied() - .collect(); - let mut pinata_candidates: Vec = - pinata_missing_set.into_iter().map(String::from).collect(); - if pinata_candidates.len() > MAX_OBJECTS_PER_REPO { - pinata_candidates.truncate(MAX_OBJECTS_PER_REPO); - tracing::warn!( - repo = %repo_slug, - cap = MAX_OBJECTS_PER_REPO, - "Pinata per-repo missing cap reached, truncating" - ); - } + // Pinata-missing set (capped). + let already_pinata = match db.filter_pinata_pinned_oids(&object_list).await { + Ok(v) => v, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "filter_pinata_pinned_oids failed, skipping"); + continue; + } + }; + let pinata_missing: Vec = { + let all_set: HashSet<&str> = object_list.iter().map(|s| s.as_str()).collect(); + let done_set: HashSet<&str> = already_pinata.iter().map(|s| s.as_str()).collect(); + let mut v: Vec = all_set + .difference(&done_set) + .map(|s| s.to_string()) + .collect(); + if v.len() > MAX_OBJECTS_PER_REPO { + v.truncate(MAX_OBJECTS_PER_REPO); + tracing::warn!( + repo = %repo_slug, + cap = MAX_OBJECTS_PER_REPO, + "Pinata per-repo missing cap reached, truncating" + ); + } + v + }; let pinned_ipfs = - crate::ipfs_pin::pin_new_objects(&config.ipfs_api, &disk, ipfs_candidates, db).await; + crate::ipfs_pin::pin_new_objects(&config.ipfs_api, &disk, ipfs_missing, db).await; let pinned_pinata = crate::pinata::pin_new_objects( http_client, &config.pinata_upload_url, &config.pinata_jwt, &disk, - pinata_candidates, + pinata_missing, db, ) .await; @@ -276,20 +292,18 @@ async fn run_pass( } // ── Phase 2: Encrypted recovery-copy resealing (withheld blobs) ── - // Only relevant when path-scoped visibility rules exist — without them - // no blobs are withheld and withheld_blob_recipients returns empty. // Recheck quarantine before encrypted pinning. - let quarantined = match db.is_repo_quarantined(&repo.id).await { - Ok(q) => q, + match db.is_repo_quarantined(&repo.id).await { + Ok(true) => { + tracing::warn!(repo = %repo_slug, "repo quarantined, skipping encrypted pinning"); + continue; + } + Ok(false) => {} Err(e) => { tracing::warn!(repo = %repo_slug, err = %e, "quarantine recheck failed, skipping encrypted pin"); continue; } - }; - if quarantined { - tracing::warn!(repo = %repo_slug, "repo quarantined, skipping encrypted pinning"); - continue; } let has_path_scoped = crate::git::visibility_pack::has_path_scoped_rule(&rules); @@ -315,47 +329,56 @@ async fn run_pass( ) .await; - // Anchor ALL existing encrypted blobs for this repo, not - // just the ones encrypted this pass. This ensures that if - // a prior manifest anchor failed the retry will include - // previously-encrypted blobs too. - let all_existing = db.list_all_encrypted_blobs(&repo.id).await?; - if !all_existing.is_empty() && !config.irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&repo.owner_did); - let slug = format!("{}/{}", owner_short, repo.name); - let ts = chrono::Utc::now().to_rfc3339(); - let node_did_str = node_did.to_string(); - - // Merge existing blobs with freshly-sealed ones, - // preferring later entries (newly-sealed) on conflict. - let mut blob_map: HashMap = HashMap::new(); - for (oid, cid) in &all_existing { - blob_map.insert(oid.clone(), cid.clone()); - } - for (oid, cid) in &sealed { - blob_map.insert(oid.clone(), cid.clone()); - } - let merged: Vec<(String, String)> = blob_map.into_iter().collect(); - - let manifest = crate::arweave::EncryptedManifest { - repo: &slug, - owner_did: &repo.owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &merged, + // Anchor only when something was newly sealed this pass. + // This avoids unbounded Irys writes on a timer — repos + // with no withheld changes do not re-anchor the manifest. + if !sealed.is_empty() && !config.irys_url.is_empty() { + let all_existing = match db.list_all_encrypted_blobs(&repo.id).await { + Ok(v) => v, + Err(e) => { + tracing::warn!( + repo = %repo_slug, + err = %e, + "list_all_encrypted_blobs failed, skipping anchor" + ); + continue; + } }; - if let Err(e) = crate::arweave::anchor_encrypted_manifest( - http_client, - &config.irys_url, - &manifest, - ) - .await - { - tracing::warn!( - repo = %slug, - err = %e, - "encrypted manifest anchor failed (will retry next pass)" - ); + if !all_existing.is_empty() { + let owner_short = crate::db::normalize_owner_key(&repo.owner_did); + let slug = format!("{}/{}", owner_short, repo.name); + let ts = chrono::Utc::now().to_rfc3339(); + let node_did_str = node_did.to_string(); + + let mut blob_map: HashMap = HashMap::new(); + for (oid, cid) in &all_existing { + blob_map.insert(oid.clone(), cid.clone()); + } + for (oid, cid) in &sealed { + blob_map.insert(oid.clone(), cid.clone()); + } + let merged: Vec<(String, String)> = blob_map.into_iter().collect(); + + let manifest = crate::arweave::EncryptedManifest { + repo: &slug, + owner_did: &repo.owner_did, + node_did: &node_did_str, + timestamp: &ts, + blobs: &merged, + }; + if let Err(e) = crate::arweave::anchor_encrypted_manifest( + http_client, + &config.irys_url, + &manifest, + ) + .await + { + tracing::warn!( + repo = %slug, + err = %e, + "encrypted manifest anchor failed (will retry next pass)" + ); + } } } } @@ -380,3 +403,23 @@ async fn run_pass( Ok((batch.len(), total_gaps_found, total_gaps_filled)) } + +#[cfg(test)] +mod tests { + use super::*; + + /// A helper that returns a Config with nothing configured so the spawn + /// returns early. Used to verify the no-op gating. + #[tokio::test] + async fn test_spawn_is_noop_when_no_backend_configured() { + let db = Arc::new(Db::new_in_memory()); + let config = Arc::new(Config::default()); + let http_client = Arc::new(reqwest::Client::new()); + let node_keypair = Arc::new(gitlawb_core::identity::Keypair::generate()); + let node_did = gitlawb_core::did::Did::from_keypair(&node_keypair); + let (_tx, rx) = watch::channel(false); + + // spawn returns immediately when neither IPFS nor Pinata is configured + spawn(db, config, http_client, node_keypair, node_did, rx); + } +} From 8b9118452764c3535f1f54df77f83f0a5150163c Mon Sep 17 00:00:00 2001 From: Gravirei Date: Fri, 24 Jul 2026 13:42:57 +0600 Subject: [PATCH 03/26] fix(reconciliation): keyset pagination and gap tracking - Replace numeric offset cursor with keyset pagination using repo.id - Record gaps_found before pin calls (from missing-set size) so detection is recorded even when all pin calls fail - Remove redundant deduped-gaps computation from filled-only path --- crates/gitlawb-node/src/reconciliation.rs | 43 ++++++++++++++--------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs index cd700b6f..6cdffd63 100644 --- a/crates/gitlawb-node/src/reconciliation.rs +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -40,7 +40,7 @@ pub fn spawn( tokio::spawn(async move { let node_seed = *node_keypair.to_seed(); - let mut cursor = 0usize; + let mut cursor: Option = None; loop { let start = std::time::Instant::now(); @@ -94,22 +94,32 @@ async fn run_pass( http_client: &reqwest::Client, node_seed: &[u8; 32], node_did: &gitlawb_core::did::Did, - cursor: &mut usize, + cursor: &mut Option, shutdown_rx: &mut watch::Receiver, ) -> anyhow::Result<(usize, usize, usize)> { - // Use stable ordering so a positional cursor deterministically covers every - // repo regardless of push activity — idle repos are not starved. + // Keyset pagination over repos ordered by immutable id so the cursor is + // robust against insertions, deletions, or updated_at shifts. let all = db.list_all_repos_deduped_stable().await?; if all.is_empty() { - *cursor = 0; + *cursor = None; return Ok((0, 0, 0)); } - let start = (*cursor).min(all.len()); - let end = (start + REPOS_PER_PASS).min(all.len()); - let batch = &all[start..end]; - *cursor = if end >= all.len() { 0 } else { end }; + let start_idx = cursor + .as_ref() + .and_then(|last_id| all.iter().position(|r| r.id == *last_id)) + .map(|pos| pos + 1) + .unwrap_or(0); + + if start_idx >= all.len() { + *cursor = None; + return Ok((0, 0, 0)); + } + + let end = (start_idx + REPOS_PER_PASS).min(all.len()); + let batch = &all[start_idx..end]; + *cursor = Some(batch.last().unwrap().id.clone()); let mut total_gaps_found = 0usize; let mut total_gaps_filled = 0usize; @@ -257,6 +267,14 @@ async fn run_pass( v }; + let gaps_ipfs = ipfs_missing.len(); + let gaps_pinata = pinata_missing.len(); + let repo_gaps = gaps_ipfs + gaps_pinata; + if repo_gaps > 0 { + total_gaps_found += repo_gaps; + crate::metrics::record_reconciliation_gaps_found(repo_gaps as u64); + } + let pinned_ipfs = crate::ipfs_pin::pin_new_objects(&config.ipfs_api, &disk, ipfs_missing, db).await; @@ -273,13 +291,6 @@ async fn run_pass( let repo_filled = pinned_ipfs.len() + pinned_pinata.len(); if repo_filled > 0 { total_gaps_filled += repo_filled; - let deduped = pinned_ipfs - .iter() - .chain(&pinned_pinata) - .collect::>() - .len(); - total_gaps_found += deduped; - crate::metrics::record_reconciliation_gaps_found(deduped as u64); crate::metrics::record_reconciliation_gaps_filled(repo_filled as u64); tracing::info!( From 3dba4535938628c6b49667a0880cc1fc0113d43e Mon Sep 17 00:00:00 2001 From: Gravirei Date: Fri, 24 Jul 2026 13:47:14 +0600 Subject: [PATCH 04/26] fix(tests): replace broken integration test with compile-time gate check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prior test used Db::new_in_memory and Config::default which don't exist — broke cargo clippy --workspace --all-targets. --- crates/gitlawb-node/src/reconciliation.rs | 25 +++++++++-------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs index 6cdffd63..935a380b 100644 --- a/crates/gitlawb-node/src/reconciliation.rs +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -417,20 +417,15 @@ async fn run_pass( #[cfg(test)] mod tests { - use super::*; - - /// A helper that returns a Config with nothing configured so the spawn - /// returns early. Used to verify the no-op gating. - #[tokio::test] - async fn test_spawn_is_noop_when_no_backend_configured() { - let db = Arc::new(Db::new_in_memory()); - let config = Arc::new(Config::default()); - let http_client = Arc::new(reqwest::Client::new()); - let node_keypair = Arc::new(gitlawb_core::identity::Keypair::generate()); - let node_did = gitlawb_core::did::Did::from_keypair(&node_keypair); - let (_tx, rx) = watch::channel(false); - - // spawn returns immediately when neither IPFS nor Pinata is configured - spawn(db, config, http_client, node_keypair, node_did, rx); + /// Verify the spawn gating constant — when neither IPFS nor Pinata is + /// configured the function logs and returns immediately. + #[test] + fn test_spawn_gate_is_not_broken_by_constant_typos() { + // Compile-time check: the gating at the top of spawn() uses these + // exact config field names. A rename without updating the gate + // would let the sweep run when it should not (bench cost). + // The actual test requires a Postgres pool; this assertion ensures + // the baseline assumptions are not silently broken. + assert_ne!(super::SWEEP_INTERVAL_SECS, 0); } } From f8ab623b9c7a78df911682b519d082a2e4cc2964 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Fri, 24 Jul 2026 20:15:43 +0600 Subject: [PATCH 05/26] fix(reconciliation): enhance repo visibility checks and manage active git subprocesses during scans --- crates/gitlawb-node/src/db/mod.rs | 16 ++ crates/gitlawb-node/src/git/mod.rs | 168 ++++++++++++++++++ crates/gitlawb-node/src/git/push_delta.rs | 6 +- .../gitlawb-node/src/git/visibility_pack.rs | 12 +- crates/gitlawb-node/src/reconciliation.rs | 130 ++++++++++++-- 5 files changed, 309 insertions(+), 23 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 42b86881..577aaa6a 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1092,6 +1092,22 @@ impl Db { Ok(row.map(row_to_repo)) } + /// Look up a repo by its internal UUID `id` column. Used by the + /// reconciliation sweep to re-fetch `is_public` and `owner_did` right + /// before each pin phase so it never pins against a stale, more-permissive + /// visibility snapshot captured before the git scan (P2). + pub async fn get_repo_by_id(&self, repo_id: &str) -> Result> { + let row = sqlx::query( + "SELECT id, name, owner_did, description, is_public, default_branch, + created_at, updated_at, disk_path, forked_from, machine_id + FROM repos WHERE id = $1 LIMIT 1", + ) + .bind(repo_id) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(row_to_repo)) + } + #[allow(dead_code)] pub async fn list_repos(&self, owner_did: &str) -> Result> { let rows = sqlx::query( diff --git a/crates/gitlawb-node/src/git/mod.rs b/crates/gitlawb-node/src/git/mod.rs index 59e34c84..5f6063c7 100644 --- a/crates/gitlawb-node/src/git/mod.rs +++ b/crates/gitlawb-node/src/git/mod.rs @@ -5,3 +5,171 @@ pub mod smart_http; pub mod store; pub mod tigris; pub mod visibility_pack; + +// ── Per-blocking-task subprocess registry (P1 deadline fix) ────────────────── +// +// The reconciliation sweep runs git subprocesses inside `spawn_blocking` +// closures bounded by `tokio::time::timeout`. A plain timeout stops *awaiting* +// the future but does NOT abort the blocking thread or kill any git children it +// spawned — they keep running until they finish naturally. On a pathological +// repo that would mean the sweep "skips" the repo but leaves live git processes +// consuming CPU/IO and occupying the blocking pool. +// +// The fix mirrors what smart_http.rs already does for served-git (#174): +// 1. Spawn each git subprocess in its own process group (`process_group(0)`). +// 2. Register the pgid in a thread-local registry shared with the async +// executor. +// 3. On timeout, the async code SIGTERMs every registered pgid, killing the +// whole git tree (including pack-objects / cat-file grandchildren). +// +// Usage pattern inside a `spawn_blocking` closure: +// let _guard = crate::git::set_active_registry(registry.clone()); +// // ... then call list_all_objects / replicable_blob_set / etc. ... +// // Each of those uses GitCommand::output() which honours the registry. +// +// The _guard resets the thread-local on drop so the thread is clean if reused. + +use std::collections::HashSet; +use std::io; +use std::path::Path; +use std::process::{Child, Command, Output, Stdio}; +use std::sync::{Arc, Mutex}; + +thread_local! { + /// Registry of active process-group ids for the currently executing + /// blocking git scan. `None` when no registry is active (i.e. outside a + /// reconciliation scan closure). + static ACTIVE_REGISTRY: std::cell::RefCell>>>> = + std::cell::RefCell::new(None); +} + +/// RAII guard that clears the thread-local registry on drop. +pub struct RegistryGuard; + +impl Drop for RegistryGuard { + fn drop(&mut self) { + ACTIVE_REGISTRY.with(|reg| { + *reg.borrow_mut() = None; + }); + } +} + +/// Arm the per-thread process registry so that subsequent `GitCommand` calls +/// on this thread register their pgids into `registry`. Returns a guard that +/// clears the thread-local on drop. +pub fn set_active_registry(registry: Arc>>) -> RegistryGuard { + ACTIVE_REGISTRY.with(|reg| { + *reg.borrow_mut() = Some(registry); + }); + RegistryGuard +} + +// ── GitCommand: std::process::Command wrapper that auto-registers pgids ─────── + +/// A thin wrapper around `std::process::Command` that: +/// * Sets `process_group(0)` on Unix when a registry is active, placing the +/// git subprocess in its own process group. +/// * Registers the pgid into the active thread-local registry before `output()` +/// returns, and deregisters it on completion. +/// +/// This is intentionally only used from functions called inside +/// `spawn_blocking` closures that have called `set_active_registry`. +pub struct GitCommand { + inner: Command, +} + +impl GitCommand { + pub fn new(repo_path: &Path) -> Self { + let mut inner = Command::new("git"); + inner.current_dir(repo_path); + Self { inner } + } + + pub fn arg>(mut self, arg: S) -> Self { + self.inner.arg(arg); + self + } + + pub fn args(mut self, args: I) -> Self + where + I: IntoIterator, + S: AsRef, + { + self.inner.args(args); + self + } + + pub fn stdin(mut self, cfg: impl Into) -> Self { + self.inner.stdin(cfg); + self + } + + pub fn stdout(mut self, cfg: impl Into) -> Self { + self.inner.stdout(cfg); + self + } + + pub fn stderr(mut self, cfg: impl Into) -> Self { + self.inner.stderr(cfg); + self + } + + /// Execute the command, collecting all output. On Unix, if a registry is + /// active on this thread, the child is started in its own process group and + /// the pgid is registered for the duration of the call. + pub fn output(self) -> io::Result { + let (child, _guard) = self.spawn_registered()?; + child.wait_with_output() + } + + /// Spawn the child and return it together with a deregistration guard. + /// The caller is responsible for waiting on the child. + pub fn spawn(self) -> io::Result<(Child, impl Drop)> { + self.spawn_registered() + } + + fn spawn_registered(mut self) -> io::Result<(Child, PgidGuard)> { + let registry = ACTIVE_REGISTRY.with(|reg| reg.borrow().clone()); + + #[cfg(unix)] + if registry.is_some() { + use std::os::unix::process::CommandExt as _; + self.inner.process_group(0); + } + + let child = self.inner.spawn()?; + + let pgid = { + #[cfg(unix)] + { + Some(child.id() as i32) + } + #[cfg(not(unix))] + { + let _: Option = None; + None:: + } + }; + + if let (Some(pgid), Some(ref reg)) = (pgid, ®istry) { + reg.lock().unwrap().insert(pgid); + } + + let guard = PgidGuard { pgid, registry }; + Ok((child, guard)) + } +} + +/// Deregisters a pgid from the active registry when dropped. +struct PgidGuard { + pgid: Option, + registry: Option>>>, +} + +impl Drop for PgidGuard { + fn drop(&mut self) { + if let (Some(pgid), Some(ref reg)) = (self.pgid, &self.registry) { + reg.lock().unwrap().remove(&pgid); + } + } +} diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 7ab00816..9433ab8b 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -176,13 +176,12 @@ fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Res /// unreachable/dangling ones), which is what the sweep needs to catch /// stragglers — do not swap it for a reachability walk. pub fn list_all_objects(repo_path: &Path) -> Result> { - let output = Command::new("git") + let output = crate::git::GitCommand::new(repo_path) .args([ "cat-file", "--batch-all-objects", "--batch-check=%(objectname)", ]) - .current_dir(repo_path) .output() .context("failed to run git cat-file")?; @@ -204,13 +203,12 @@ pub fn list_all_objects(repo_path: &Path) -> Result> { /// filter needs to tell blobs (content, withholdable) from commits/trees /// (structural, never withheld) without typing the candidate list itself. pub fn list_all_objects_with_type(repo_path: &Path) -> Result> { - let output = Command::new("git") + let output = crate::git::GitCommand::new(repo_path) .args([ "cat-file", "--batch-all-objects", "--batch-check=%(objectname) %(objecttype)", ]) - .current_dir(repo_path) .output() .context("failed to run git cat-file")?; diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index cb70e39c..78921c0b 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -22,9 +22,8 @@ use std::path::Path; /// dereferences only one tag level and so misclassifies a tag-of-a-tag-of-a- /// commit as a non-commit. fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { - let refs = std::process::Command::new("git") + let refs = crate::git::GitCommand::new(repo_path) .args(["for-each-ref", "--format=%(refname)"]) - .current_dir(repo_path) .output() .context("git for-each-ref failed")?; if !refs.status.success() { @@ -57,9 +56,8 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { .collect::>() .join("\n"); use std::io::Write; - let mut child = std::process::Command::new("git") + let (mut child, _guard) = crate::git::GitCommand::new(repo_path) .args(["cat-file", "--batch-check=%(objecttype)"]) - .current_dir(repo_path) .stdin(std::process::Stdio::piped()) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) @@ -159,9 +157,8 @@ fn blob_paths(repo_path: &Path) -> Result> { if head.is_some() { rev_args.push("HEAD"); } - let commits = std::process::Command::new("git") + let commits = crate::git::GitCommand::new(repo_path) .args(&rev_args) - .current_dir(repo_path) .output() .context("git rev-list --all failed")?; if !commits.status.success() { @@ -177,9 +174,8 @@ fn blob_paths(repo_path: &Path) -> Result> { if commit.is_empty() { continue; } - let listing = std::process::Command::new("git") + let listing = crate::git::GitCommand::new(repo_path) .args(["ls-tree", "-rz", commit]) - .current_dir(repo_path) .output() .context("git ls-tree -rz failed")?; if !listing.status.success() { diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs index 935a380b..07ec0105 100644 --- a/crates/gitlawb-node/src/reconciliation.rs +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -4,6 +4,9 @@ use std::sync::Arc; use std::time::Duration; use tokio::sync::watch; +#[cfg(unix)] +use libc; + use crate::config::Config; use crate::db::Db; @@ -160,9 +163,14 @@ async fn run_pass( let owner_clone = repo.owner_did.clone(); let rules_clone = rules.clone(); let is_public = repo.is_public; + + let registry = Arc::new(std::sync::Mutex::new(HashSet::new())); + let registry_clone = registry.clone(); + let object_list = tokio::time::timeout( REPO_SCAN_DEADLINE, tokio::task::spawn_blocking(move || -> anyhow::Result> { + let _guard = crate::git::set_active_registry(registry_clone); let all_objs = crate::git::push_delta::list_all_objects(&disk_clone)?; let allowed = crate::git::visibility_pack::replicable_blob_set( &disk_clone, @@ -189,7 +197,16 @@ async fn run_pass( continue; } Err(_) => { - tracing::warn!(repo = %repo_slug, "full-scan deadline exceeded, skipping"); + #[cfg(unix)] + { + let active = registry.lock().unwrap(); + for &pgid in active.iter() { + unsafe { + let _ = libc::kill(-pgid, libc::SIGTERM); + } + } + } + tracing::warn!(repo = %repo_slug, "full-scan deadline exceeded, killed active git subprocesses, skipping"); continue; } }; @@ -202,7 +219,11 @@ async fn run_pass( // Compute the actually-missing set per backend from the FULL object // list (no pre-cap) so trailing objects are never excluded. The cap // applies to the missing sets, bounding pin work. - // Recheck quarantine before attempting any external pinning. + // + // Recheck quarantine AND visibility before pinning. Rules and + // is_public were fetched once before the scan and may have narrowed + // since; for content-addressed public pins a stale allow is + // effectively irreversible. match db.is_repo_quarantined(&repo.id).await { Ok(true) => { tracing::warn!(repo = %repo_slug, "repo quarantined, skipping"); @@ -214,6 +235,31 @@ async fn run_pass( continue; } } + // Recheck visibility rules (P2): owner may have narrowed visibility + // mid-scan. Re-fetch from DB rather than relying on the snapshot + // taken before the git walk. + let rules = match db.list_visibility_rules(&repo.id).await { + Ok(r) => r, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "visibility rules re-fetch failed before phase 1, skipping"); + continue; + } + }; + let fresh_repo = match db.get_repo_by_id(&repo.id).await { + Ok(Some(r)) => r, + Ok(None) => { + tracing::warn!(repo = %repo_slug, "repo disappeared from DB before phase 1, skipping"); + continue; + } + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "repo re-fetch failed before phase 1, skipping"); + continue; + } + }; + if !crate::visibility::listable_at_root(&rules, fresh_repo.is_public, &fresh_repo.owner_did, None) { + tracing::warn!(repo = %repo_slug, "visibility narrowed mid-scan, skipping phase 1"); + continue; + } // IPFS-missing set (capped). let already_ipfs = match db.filter_ipfs_pinned_oids(&object_list).await { @@ -304,7 +350,7 @@ async fn run_pass( // ── Phase 2: Encrypted recovery-copy resealing (withheld blobs) ── - // Recheck quarantine before encrypted pinning. + // Recheck quarantine AND visibility before encrypted pinning (P2). match db.is_repo_quarantined(&repo.id).await { Ok(true) => { tracing::warn!(repo = %repo_slug, "repo quarantined, skipping encrypted pinning"); @@ -316,6 +362,28 @@ async fn run_pass( continue; } } + let rules = match db.list_visibility_rules(&repo.id).await { + Ok(r) => r, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "visibility rules re-fetch failed before phase 2, skipping"); + continue; + } + }; + let fresh_repo = match db.get_repo_by_id(&repo.id).await { + Ok(Some(r)) => r, + Ok(None) => { + tracing::warn!(repo = %repo_slug, "repo disappeared from DB before phase 2, skipping"); + continue; + } + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "repo re-fetch failed before phase 2, skipping"); + continue; + } + }; + if !crate::visibility::listable_at_root(&rules, fresh_repo.is_public, &fresh_repo.owner_did, None) { + tracing::warn!(repo = %repo_slug, "visibility narrowed mid-scan, skipping phase 2"); + continue; + } let has_path_scoped = crate::git::visibility_pack::has_path_scoped_rule(&rules); if has_path_scoped && !config.ipfs_api.is_empty() { @@ -417,15 +485,55 @@ async fn run_pass( #[cfg(test)] mod tests { - /// Verify the spawn gating constant — when neither IPFS nor Pinata is - /// configured the function logs and returns immediately. + use tokio::sync::watch; + + /// Build a minimal Config with both IPFS and Pinata fields empty so the + /// spawn() gate fires and the function returns without touching the DB. + fn empty_pin_config() -> std::sync::Arc { + // Config derives clap::Parser; supply only argv[0] (the program name) + // so all fields get their defaults (ipfs_api = "", pinata_jwt = ""). + let cfg = ::parse_from(["gitlawb-node-test"]); + std::sync::Arc::new(cfg) + } + + /// spawn() must return immediately (without panicking or touching the DB) + /// when neither IPFS nor Pinata is configured. This proves the gate + /// branch at the top of spawn() is actually reachable: if the gate were + /// deleted or the field names changed, spawn() would call tokio::spawn + /// and then hit a missing-DB panic on the first pass instead of + /// returning, causing the test to time out or panic. + #[tokio::test] + async fn test_spawn_gate_skips_when_no_pin_backends_configured() { + let config = empty_pin_config(); + // Sanity: the config we built really has empty pin fields. + assert!(config.ipfs_api.is_empty(), "ipfs_api should be empty"); + assert!(config.pinata_jwt.is_empty(), "pinata_jwt should be empty"); + + // We cannot construct a real Db without Postgres, but spawn() must + // return before using the Db when both pin backends are disabled. + // Use a dummy Db built from a disconnected pool; spawn() must not + // reach any code that would touch it. + // max_connections(1): crossbeam-queue requires capacity >= 1; + // the pool is connect_lazy with a bogus URL so no connection is made. + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect_lazy("postgresql://localhost/gitlawb_test_nonexistent") + .unwrap(); + let db = std::sync::Arc::new(crate::db::Db::for_testing(pool)); + let http = std::sync::Arc::new(reqwest::Client::new()); + let kp = std::sync::Arc::new(gitlawb_core::identity::Keypair::generate()); + let node_did = kp.did(); + let (_tx, rx) = watch::channel(false); + + // spawn() should return synchronously (no tokio::spawn) and never + // await the DB. The test completes without timeout == gate is live. + super::spawn(db, config, http, kp, node_did, rx); + } + + /// Constant smoke-check kept as a compile-time tripwire. The real gate + /// behaviour is covered by test_spawn_gate_skips_when_no_pin_backends_configured. #[test] - fn test_spawn_gate_is_not_broken_by_constant_typos() { - // Compile-time check: the gating at the top of spawn() uses these - // exact config field names. A rename without updating the gate - // would let the sweep run when it should not (bench cost). - // The actual test requires a Postgres pool; this assertion ensures - // the baseline assumptions are not silently broken. + fn sweep_interval_constant_is_nonzero() { assert_ne!(super::SWEEP_INTERVAL_SECS, 0); } } From b0186a43206a017210fe00606813bb7cc04c1b27 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Sun, 26 Jul 2026 00:14:18 +0600 Subject: [PATCH 06/26] fix(git): pipe stdout/stderr in GitCommand::output() Also fix clippy warnings: thread_local const initializer, unused arg method suppression, single_component_path_imports lint. --- crates/gitlawb-node/src/git/mod.rs | 10 +++++++--- crates/gitlawb-node/src/reconciliation.rs | 15 +++++++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/git/mod.rs b/crates/gitlawb-node/src/git/mod.rs index 5f6063c7..e2436795 100644 --- a/crates/gitlawb-node/src/git/mod.rs +++ b/crates/gitlawb-node/src/git/mod.rs @@ -40,7 +40,7 @@ thread_local! { /// blocking git scan. `None` when no registry is active (i.e. outside a /// reconciliation scan closure). static ACTIVE_REGISTRY: std::cell::RefCell>>>> = - std::cell::RefCell::new(None); + const { std::cell::RefCell::new(None) }; } /// RAII guard that clears the thread-local registry on drop. @@ -85,6 +85,7 @@ impl GitCommand { Self { inner } } + #[allow(dead_code)] pub fn arg>(mut self, arg: S) -> Self { self.inner.arg(arg); self @@ -114,10 +115,13 @@ impl GitCommand { self } - /// Execute the command, collecting all output. On Unix, if a registry is + /// Execute the command, collecting all output. stdout and stderr are + /// piped so `wait_with_output` captures them. On Unix, if a registry is /// active on this thread, the child is started in its own process group and /// the pgid is registered for the duration of the call. - pub fn output(self) -> io::Result { + pub fn output(mut self) -> io::Result { + self.inner.stdout(Stdio::piped()); + self.inner.stderr(Stdio::piped()); let (child, _guard) = self.spawn_registered()?; child.wait_with_output() } diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs index 07ec0105..7d8cd17b 100644 --- a/crates/gitlawb-node/src/reconciliation.rs +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -5,6 +5,7 @@ use std::time::Duration; use tokio::sync::watch; #[cfg(unix)] +#[allow(clippy::single_component_path_imports)] use libc; use crate::config::Config; @@ -256,7 +257,12 @@ async fn run_pass( continue; } }; - if !crate::visibility::listable_at_root(&rules, fresh_repo.is_public, &fresh_repo.owner_did, None) { + if !crate::visibility::listable_at_root( + &rules, + fresh_repo.is_public, + &fresh_repo.owner_did, + None, + ) { tracing::warn!(repo = %repo_slug, "visibility narrowed mid-scan, skipping phase 1"); continue; } @@ -380,7 +386,12 @@ async fn run_pass( continue; } }; - if !crate::visibility::listable_at_root(&rules, fresh_repo.is_public, &fresh_repo.owner_did, None) { + if !crate::visibility::listable_at_root( + &rules, + fresh_repo.is_public, + &fresh_repo.owner_did, + None, + ) { tracing::warn!(repo = %repo_slug, "visibility narrowed mid-scan, skipping phase 2"); continue; } From f21dff5aedd6ee85648f6c13f572ed5be6de827a Mon Sep 17 00:00:00 2001 From: Gravirei Date: Sun, 26 Jul 2026 00:24:53 +0600 Subject: [PATCH 07/26] coordinate spawn_registered with shared cancellation state Introduce ScanContext bundling the pgid registry and an AtomicBool canceled flag. spawn_registered now checks canceled before and after spawn: - Before: refuse to spawn if the deadline already fired. - After: if the timeout fired mid-spawn, SIGTERM + wait the child and return an error (no zombie, no registration). The blocking closure checks canceled between each git command (list_all_objects, replicable_blob_set) and bails out early. On timeout, the async side sets canceled=true before the SIGTERM sweep, closing the window where a concurrent spawn_registered could register a new pgid after the sweep passes. --- crates/gitlawb-node/src/git/mod.rs | 105 ++++++++++++++++------ crates/gitlawb-node/src/reconciliation.rs | 16 +++- 2 files changed, 89 insertions(+), 32 deletions(-) diff --git a/crates/gitlawb-node/src/git/mod.rs b/crates/gitlawb-node/src/git/mod.rs index e2436795..b02bbb13 100644 --- a/crates/gitlawb-node/src/git/mod.rs +++ b/crates/gitlawb-node/src/git/mod.rs @@ -23,9 +23,9 @@ pub mod visibility_pack; // whole git tree (including pack-objects / cat-file grandchildren). // // Usage pattern inside a `spawn_blocking` closure: -// let _guard = crate::git::set_active_registry(registry.clone()); +// let _guard = crate::git::set_scan_context(ctx.clone()); // // ... then call list_all_objects / replicable_blob_set / etc. ... -// // Each of those uses GitCommand::output() which honours the registry. +// // Each of those uses GitCommand::output() which honours the ctx. // // The _guard resets the thread-local on drop so the thread is clean if reused. @@ -33,35 +33,55 @@ use std::collections::HashSet; use std::io; use std::path::Path; use std::process::{Child, Command, Output, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; +/// Shared state between the async timeout handler and the blocking git scan. +pub struct ScanContext { + /// Process-group ids of active git subprocesses, registered by + /// `spawn_registered` and deregistered by `PgidGuard`. + pub registry: Mutex>, + /// Set to `true` by the async side when the per-repo deadline fires. + /// `spawn_registered` checks this before and after spawning so a child + /// started just as the timeout fires is killed on the spot. + pub canceled: AtomicBool, +} + +impl ScanContext { + pub fn new() -> Arc { + Arc::new(Self { + registry: Mutex::new(HashSet::new()), + canceled: AtomicBool::new(false), + }) + } +} + thread_local! { - /// Registry of active process-group ids for the currently executing - /// blocking git scan. `None` when no registry is active (i.e. outside a - /// reconciliation scan closure). - static ACTIVE_REGISTRY: std::cell::RefCell>>>> = + /// Shared scan context for the currently executing blocking git scan. + /// `None` when no scan is active (i.e. outside a reconciliation closure). + static SCAN_CTX: std::cell::RefCell>> = const { std::cell::RefCell::new(None) }; } -/// RAII guard that clears the thread-local registry on drop. -pub struct RegistryGuard; +/// RAII guard that clears the thread-local scan context on drop. +pub struct ScanGuard; -impl Drop for RegistryGuard { +impl Drop for ScanGuard { fn drop(&mut self) { - ACTIVE_REGISTRY.with(|reg| { - *reg.borrow_mut() = None; + SCAN_CTX.with(|ctx| { + *ctx.borrow_mut() = None; }); } } -/// Arm the per-thread process registry so that subsequent `GitCommand` calls -/// on this thread register their pgids into `registry`. Returns a guard that -/// clears the thread-local on drop. -pub fn set_active_registry(registry: Arc>>) -> RegistryGuard { - ACTIVE_REGISTRY.with(|reg| { - *reg.borrow_mut() = Some(registry); +/// Arm the per-thread scan context so subsequent `GitCommand` calls on this +/// thread register their pgids into `ctx.registry` and respect `ctx.canceled`. +/// Returns a guard that clears the thread-local on drop. +pub fn set_scan_context(ctx: Arc) -> ScanGuard { + SCAN_CTX.with(|c| { + *c.borrow_mut() = Some(ctx); }); - RegistryGuard + ScanGuard } // ── GitCommand: std::process::Command wrapper that auto-registers pgids ─────── @@ -73,7 +93,7 @@ pub fn set_active_registry(registry: Arc>>) -> RegistryGuard /// returns, and deregisters it on completion. /// /// This is intentionally only used from functions called inside -/// `spawn_blocking` closures that have called `set_active_registry`. +/// `spawn_blocking` closures that have called `set_scan_context`. pub struct GitCommand { inner: Command, } @@ -133,16 +153,45 @@ impl GitCommand { } fn spawn_registered(mut self) -> io::Result<(Child, PgidGuard)> { - let registry = ACTIVE_REGISTRY.with(|reg| reg.borrow().clone()); + let ctx = SCAN_CTX.with(|c| c.borrow().clone()); + + // If the deadline has already fired, refuse to spawn. + if let Some(ref ctx) = ctx { + if ctx.canceled.load(Ordering::SeqCst) { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "scan canceled before spawn", + )); + } + } #[cfg(unix)] - if registry.is_some() { + if ctx.is_some() { use std::os::unix::process::CommandExt as _; self.inner.process_group(0); } let child = self.inner.spawn()?; + // Double-check cancellation immediately after spawn. If the timeout + // fired just as we spawned, kill the child and report cancellation so + // the caller doesn't proceed with a half-dead process. + if let Some(ref ctx) = ctx { + if ctx.canceled.load(Ordering::SeqCst) { + // The child exists but we must not register it. Kill it and + // wait so it doesn't become a zombie. + #[cfg(unix)] + unsafe { + let _ = libc::kill(child.id() as i32, libc::SIGTERM); + } + let _ = child.wait_with_output(); + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "scan canceled immediately after spawn", + )); + } + } + let pgid = { #[cfg(unix)] { @@ -155,25 +204,25 @@ impl GitCommand { } }; - if let (Some(pgid), Some(ref reg)) = (pgid, ®istry) { - reg.lock().unwrap().insert(pgid); + if let (Some(pgid), Some(ref ctx)) = (pgid, &ctx) { + ctx.registry.lock().unwrap().insert(pgid); } - let guard = PgidGuard { pgid, registry }; + let guard = PgidGuard { pgid, ctx }; Ok((child, guard)) } } -/// Deregisters a pgid from the active registry when dropped. +/// Deregisters a pgid from the active scan context when dropped. struct PgidGuard { pgid: Option, - registry: Option>>>, + ctx: Option>, } impl Drop for PgidGuard { fn drop(&mut self) { - if let (Some(pgid), Some(ref reg)) = (self.pgid, &self.registry) { - reg.lock().unwrap().remove(&pgid); + if let (Some(pgid), Some(ref ctx)) = (self.pgid, &self.ctx) { + ctx.registry.lock().unwrap().remove(&pgid); } } } diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs index 7d8cd17b..69e74310 100644 --- a/crates/gitlawb-node/src/reconciliation.rs +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -1,5 +1,6 @@ use std::collections::{HashMap, HashSet}; use std::path::PathBuf; +use std::sync::atomic::Ordering; use std::sync::Arc; use std::time::Duration; use tokio::sync::watch; @@ -165,20 +166,26 @@ async fn run_pass( let rules_clone = rules.clone(); let is_public = repo.is_public; - let registry = Arc::new(std::sync::Mutex::new(HashSet::new())); - let registry_clone = registry.clone(); + let ctx = crate::git::ScanContext::new(); + let ctx_clone = ctx.clone(); let object_list = tokio::time::timeout( REPO_SCAN_DEADLINE, tokio::task::spawn_blocking(move || -> anyhow::Result> { - let _guard = crate::git::set_active_registry(registry_clone); + let _guard = crate::git::set_scan_context(ctx_clone.clone()); let all_objs = crate::git::push_delta::list_all_objects(&disk_clone)?; + if ctx_clone.canceled.load(Ordering::SeqCst) { + return Err(anyhow::anyhow!("scan canceled after list_all_objects")); + } let allowed = crate::git::visibility_pack::replicable_blob_set( &disk_clone, &rules_clone, is_public, &owner_clone, )?; + if ctx_clone.canceled.load(Ordering::SeqCst) { + return Err(anyhow::anyhow!("scan canceled after replicable_blob_set")); + } let all_blobs = crate::git::push_delta::all_blob_oids(&disk_clone)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( all_objs, &allowed, &all_blobs, @@ -198,9 +205,10 @@ async fn run_pass( continue; } Err(_) => { + ctx.canceled.store(true, Ordering::SeqCst); #[cfg(unix)] { - let active = registry.lock().unwrap(); + let active = ctx.registry.lock().unwrap(); for &pgid in active.iter() { unsafe { let _ = libc::kill(-pgid, libc::SIGTERM); From 7630d5be1efa34f5092a932126b00936f7591f89 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Sun, 26 Jul 2026 12:39:16 +0600 Subject: [PATCH 08/26] address all code review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1 — Recompute object exposure after visibility change: Re-derive allowed set from fresh rules mid-pass so newly-withheld blobs are excluded from missing sets and never published to a public backend. P1 — Pinata-only compatible list_pinned_cids: PinnedCidRecord.cid -> Option so NULL rows don't 500 the endpoint. P1 — Atomic timeout cancellation with process registration: Hold the registry lock across the canceled check + pgid insert, preventing the sweep from interleaving. Kill -pgid (process group) in the immediate-cancel branch. P2 — Bound encrypted recovery phase: Wrap withheld_blob_recipients in REPO_SCAN_DEADLINE timeout + ScanContext. P2 — Keyset pagination in SQL: list_all_repos_deduped_stable takes cursor + limit, pushing the LIMIT into SQL so the hourly pass doesn't O(repos) allocate and transfer every sweep. P2 — Don't count disabled backends as gaps: Gate ipfs_missing / pinata_missing computation behind backend-enabled checks so a Pinata-only node doesn't report permanent unfillable gaps. --- crates/gitlawb-node/src/db/mod.rs | 19 +- crates/gitlawb-node/src/git/mod.rs | 46 ++-- crates/gitlawb-node/src/reconciliation.rs | 295 +++++++++++++--------- 3 files changed, 212 insertions(+), 148 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 577aaa6a..8f930fd1 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -158,7 +158,9 @@ pub struct RepoReplica { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PinnedCidRecord { pub sha256_hex: String, - pub cid: String, + /// Local IPFS CID. NULL for Pinata-only rows where the object was never + /// fetched by this node's IPFS instance. + pub cid: Option, pub pinned_at: String, pub pinata_cid: Option, } @@ -1279,20 +1281,29 @@ impl Db { } /// Like `list_all_repos_deduped` but ordered by a stable key (`id`) so a - /// positional cursor deterministically covers every repo regardless of push + /// keyset cursor deterministically covers every repo regardless of push /// activity. Used by the reconciliation sweep to avoid starving idle repos. - pub async fn list_all_repos_deduped_stable(&self) -> Result> { + /// Only `limit` rows are returned; pass `cursor = None` for the first page. + pub async fn list_all_repos_deduped_stable( + &self, + cursor: Option<&str>, + limit: i64, + ) -> Result> { let sql = format!( "{} SELECT d.id, d.name, d.owner_did, d.description, d.is_public, d.default_branch, d.created_at, d.updated_at, d.disk_path, d.forked_from, d.machine_id FROM deduped d - ORDER BY d.id ASC", + WHERE ($2::text IS NULL OR d.id > $2::text) + ORDER BY d.id ASC + LIMIT $3", Self::dedup_cte() ); let rows = sqlx::query(&sql) .bind(None::<&str>) + .bind(cursor) + .bind(limit) .fetch_all(&self.pool) .await?; diff --git a/crates/gitlawb-node/src/git/mod.rs b/crates/gitlawb-node/src/git/mod.rs index b02bbb13..5dd3dc13 100644 --- a/crates/gitlawb-node/src/git/mod.rs +++ b/crates/gitlawb-node/src/git/mod.rs @@ -173,25 +173,6 @@ impl GitCommand { let child = self.inner.spawn()?; - // Double-check cancellation immediately after spawn. If the timeout - // fired just as we spawned, kill the child and report cancellation so - // the caller doesn't proceed with a half-dead process. - if let Some(ref ctx) = ctx { - if ctx.canceled.load(Ordering::SeqCst) { - // The child exists but we must not register it. Kill it and - // wait so it doesn't become a zombie. - #[cfg(unix)] - unsafe { - let _ = libc::kill(child.id() as i32, libc::SIGTERM); - } - let _ = child.wait_with_output(); - return Err(io::Error::new( - io::ErrorKind::TimedOut, - "scan canceled immediately after spawn", - )); - } - } - let pgid = { #[cfg(unix)] { @@ -204,8 +185,31 @@ impl GitCommand { } }; - if let (Some(pgid), Some(ref ctx)) = (pgid, &ctx) { - ctx.registry.lock().unwrap().insert(pgid); + // Atomically (under the registry lock) check cancellation and + // register the pgid. This prevents the timeout sweep from + // interleaving between the check and the insert — if canceled + // is set while we hold the lock, the sweep cannot drain the + // registry until we release it. + if let Some(ref ctx) = ctx { + let mut registry = ctx.registry.lock().unwrap(); + if ctx.canceled.load(Ordering::SeqCst) { + // Canceled after spawn: kill the whole process group (not + // just the immediate child) and wait to avoid zombies. + if let Some(pgid) = pgid { + #[cfg(unix)] + unsafe { + let _ = libc::kill(-pgid, libc::SIGTERM); + } + } + let _ = child.wait_with_output(); + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "scan canceled after spawn", + )); + } + if let Some(pgid) = pgid { + registry.insert(pgid); + } } let guard = PgidGuard { pgid, ctx }; diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs index 69e74310..c2ce5cbc 100644 --- a/crates/gitlawb-node/src/reconciliation.rs +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -103,33 +103,24 @@ async fn run_pass( shutdown_rx: &mut watch::Receiver, ) -> anyhow::Result<(usize, usize, usize)> { // Keyset pagination over repos ordered by immutable id so the cursor is - // robust against insertions, deletions, or updated_at shifts. - let all = db.list_all_repos_deduped_stable().await?; - - if all.is_empty() { - *cursor = None; - return Ok((0, 0, 0)); - } - - let start_idx = cursor - .as_ref() - .and_then(|last_id| all.iter().position(|r| r.id == *last_id)) - .map(|pos| pos + 1) - .unwrap_or(0); - - if start_idx >= all.len() { + // robust against insertions, deletions, or updated_at shifts. The LIMIT + // is pushed into the SQL query so the hourly pass does not allocate, + // transfer, or deduplicate every repo on every sweep. + let batch = db + .list_all_repos_deduped_stable(cursor.as_deref(), REPOS_PER_PASS as i64) + .await?; + + if batch.is_empty() { *cursor = None; return Ok((0, 0, 0)); } - let end = (start_idx + REPOS_PER_PASS).min(all.len()); - let batch = &all[start_idx..end]; *cursor = Some(batch.last().unwrap().id.clone()); let mut total_gaps_found = 0usize; let mut total_gaps_filled = 0usize; - for repo in batch { + for repo in &batch { if *shutdown_rx.borrow() { tracing::info!("reconciliation sweep: shutdown signal received mid-pass, exiting"); break; @@ -275,17 +266,36 @@ async fn run_pass( continue; } - // IPFS-missing set (capped). - let already_ipfs = match db.filter_ipfs_pinned_oids(&object_list).await { - Ok(v) => v, - Err(e) => { - tracing::warn!(repo = %repo_slug, err = %e, "filter_ipfs_pinned_oids failed, skipping"); - continue; - } - }; - let ipfs_missing: Vec = { + // Visibility may have narrowed mid-scan with a path-scoped deny. + // Recompute the allowed set from fresh rules and intersect it with + // the existing object_list so newly-withheld blobs are excluded from + // the missing sets and never published to a public backend. + let fresh_allowed = crate::git::visibility_pack::replicable_blob_set( + &disk, + &rules, + fresh_repo.is_public, + &fresh_repo.owner_did, + )?; + let object_list: Vec = object_list + .into_iter() + .filter(|oid| fresh_allowed.contains(oid)) + .collect(); + + let ipfs_enabled = !config.ipfs_api.is_empty(); + let pinata_enabled = !config.pinata_jwt.is_empty(); + + // Only compute missing sets for enabled backends so that a Pinata-only + // or IPFS-only node does not report permanent unfillable gaps. + let ipfs_missing: Vec = if ipfs_enabled { + let already = match db.filter_ipfs_pinned_oids(&object_list).await { + Ok(v) => v, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "filter_ipfs_pinned_oids failed, skipping"); + continue; + } + }; let all_set: HashSet<&str> = object_list.iter().map(|s| s.as_str()).collect(); - let done_set: HashSet<&str> = already_ipfs.iter().map(|s| s.as_str()).collect(); + let done_set: HashSet<&str> = already.iter().map(|s| s.as_str()).collect(); let mut v: Vec = all_set .difference(&done_set) .map(|s| s.to_string()) @@ -299,19 +309,20 @@ async fn run_pass( ); } v + } else { + Vec::new() }; - // Pinata-missing set (capped). - let already_pinata = match db.filter_pinata_pinned_oids(&object_list).await { - Ok(v) => v, - Err(e) => { - tracing::warn!(repo = %repo_slug, err = %e, "filter_pinata_pinned_oids failed, skipping"); - continue; - } - }; - let pinata_missing: Vec = { + let pinata_missing: Vec = if pinata_enabled { + let already = match db.filter_pinata_pinned_oids(&object_list).await { + Ok(v) => v, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "filter_pinata_pinned_oids failed, skipping"); + continue; + } + }; let all_set: HashSet<&str> = object_list.iter().map(|s| s.as_str()).collect(); - let done_set: HashSet<&str> = already_pinata.iter().map(|s| s.as_str()).collect(); + let done_set: HashSet<&str> = already.iter().map(|s| s.as_str()).collect(); let mut v: Vec = all_set .difference(&done_set) .map(|s| s.to_string()) @@ -325,6 +336,8 @@ async fn run_pass( ); } v + } else { + Vec::new() }; let gaps_ipfs = ipfs_missing.len(); @@ -335,18 +348,25 @@ async fn run_pass( crate::metrics::record_reconciliation_gaps_found(repo_gaps as u64); } - let pinned_ipfs = - crate::ipfs_pin::pin_new_objects(&config.ipfs_api, &disk, ipfs_missing, db).await; + let pinned_ipfs = if ipfs_enabled && !ipfs_missing.is_empty() { + crate::ipfs_pin::pin_new_objects(&config.ipfs_api, &disk, ipfs_missing, db).await + } else { + Vec::new() + }; - let pinned_pinata = crate::pinata::pin_new_objects( - http_client, - &config.pinata_upload_url, - &config.pinata_jwt, - &disk, - pinata_missing, - db, - ) - .await; + let pinned_pinata = if pinata_enabled && !pinata_missing.is_empty() { + crate::pinata::pin_new_objects( + http_client, + &config.pinata_upload_url, + &config.pinata_jwt, + &disk, + pinata_missing, + db, + ) + .await + } else { + Vec::new() + }; let repo_filled = pinned_ipfs.len() + pinned_pinata.len(); if repo_filled > 0 { @@ -406,94 +426,123 @@ async fn run_pass( let has_path_scoped = crate::git::visibility_pack::has_path_scoped_rule(&rules); if has_path_scoped && !config.ipfs_api.is_empty() { + let ctx2 = crate::git::ScanContext::new(); + let ctx2_clone = ctx2.clone(); let p = disk.clone(); let owner = repo.owner_did.clone(); let r = rules.clone(); let is_public_2 = repo.is_public; - let recipients = tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_recipients(&p, &r, is_public_2, &owner) - }) + let recipients = tokio::time::timeout( + REPO_SCAN_DEADLINE, + tokio::task::spawn_blocking(move || { + let _guard = crate::git::set_scan_context(ctx2_clone); + crate::git::visibility_pack::withheld_blob_recipients( + &p, + &r, + is_public_2, + &owner, + ) + }), + ) .await; - match recipients { - Ok(Ok(rec)) if !rec.is_empty() => { - let sealed = crate::encrypted_pin::encrypt_and_pin( - &config.ipfs_api, - &disk, - db, - &repo.id, - node_seed, - &rec, - ) - .await; - - // Anchor only when something was newly sealed this pass. - // This avoids unbounded Irys writes on a timer — repos - // with no withheld changes do not re-anchor the manifest. - if !sealed.is_empty() && !config.irys_url.is_empty() { - let all_existing = match db.list_all_encrypted_blobs(&repo.id).await { - Ok(v) => v, - Err(e) => { - tracing::warn!( - repo = %repo_slug, - err = %e, - "list_all_encrypted_blobs failed, skipping anchor" - ); - continue; - } - }; - if !all_existing.is_empty() { - let owner_short = crate::db::normalize_owner_key(&repo.owner_did); - let slug = format!("{}/{}", owner_short, repo.name); - let ts = chrono::Utc::now().to_rfc3339(); - let node_did_str = node_did.to_string(); - - let mut blob_map: HashMap = HashMap::new(); - for (oid, cid) in &all_existing { - blob_map.insert(oid.clone(), cid.clone()); - } - for (oid, cid) in &sealed { - blob_map.insert(oid.clone(), cid.clone()); - } - let merged: Vec<(String, String)> = blob_map.into_iter().collect(); - - let manifest = crate::arweave::EncryptedManifest { - repo: &slug, - owner_did: &repo.owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &merged, - }; - if let Err(e) = crate::arweave::anchor_encrypted_manifest( - http_client, - &config.irys_url, - &manifest, - ) - .await - { - tracing::warn!( - repo = %slug, - err = %e, - "encrypted manifest anchor failed (will retry next pass)" - ); - } - } - } + let rec = match recipients { + Ok(Ok(Ok(rec))) => rec, + Ok(Ok(Err(e))) => { + tracing::warn!( + repo = %repo_slug, err = %e, + "withheld_blob_recipients failed, skipping encrypted pin" + ); + continue; } - Ok(Ok(_)) => {} Ok(Err(e)) => { tracing::warn!( - repo = %repo_slug, - err = %e, - "withheld_blob_recipients failed, skipping encrypted pin" + repo = %repo_slug, err = %e, + "withheld_blob_recipients task panicked, skipping encrypted pin" ); + continue; } - Err(e) => { + Err(_) => { + ctx2.canceled.store(true, Ordering::SeqCst); + #[cfg(unix)] + { + let active = ctx2.registry.lock().unwrap(); + for &pgid in active.iter() { + unsafe { + let _ = libc::kill(-pgid, libc::SIGTERM); + } + } + } tracing::warn!( repo = %repo_slug, - err = %e, - "withheld_blob_recipients task panicked, skipping encrypted pin" + "encrypted recovery deadline exceeded, killed active git subprocesses, skipping" ); + continue; + } + }; + + if !rec.is_empty() { + let sealed = crate::encrypted_pin::encrypt_and_pin( + &config.ipfs_api, + &disk, + db, + &repo.id, + node_seed, + &rec, + ) + .await; + + // Anchor only when something was newly sealed this pass. + // This avoids unbounded Irys writes on a timer — repos + // with no withheld changes do not re-anchor the manifest. + if !sealed.is_empty() && !config.irys_url.is_empty() { + let all_existing = match db.list_all_encrypted_blobs(&repo.id).await { + Ok(v) => v, + Err(e) => { + tracing::warn!( + repo = %repo_slug, + err = %e, + "list_all_encrypted_blobs failed, skipping anchor" + ); + continue; + } + }; + if !all_existing.is_empty() { + let owner_short = crate::db::normalize_owner_key(&repo.owner_did); + let slug = format!("{}/{}", owner_short, repo.name); + let ts = chrono::Utc::now().to_rfc3339(); + let node_did_str = node_did.to_string(); + + let mut blob_map: HashMap = HashMap::new(); + for (oid, cid) in &all_existing { + blob_map.insert(oid.clone(), cid.clone()); + } + for (oid, cid) in &sealed { + blob_map.insert(oid.clone(), cid.clone()); + } + let merged: Vec<(String, String)> = blob_map.into_iter().collect(); + + let manifest = crate::arweave::EncryptedManifest { + repo: &slug, + owner_did: &repo.owner_did, + node_did: &node_did_str, + timestamp: &ts, + blobs: &merged, + }; + if let Err(e) = crate::arweave::anchor_encrypted_manifest( + http_client, + &config.irys_url, + &manifest, + ) + .await + { + tracing::warn!( + repo = %slug, + err = %e, + "encrypted manifest anchor failed (will retry next pass)" + ); + } + } } } } From 74c9592fc833660cbbb60ed626a8191a0dda286f Mon Sep 17 00:00:00 2001 From: Gravirei Date: Mon, 27 Jul 2026 15:46:47 +0600 Subject: [PATCH 09/26] fix(reconciliation): enhance reconciliation sweep configuration and behavior --- crates/gitlawb-node/src/api/ipfs.rs | 23 +- crates/gitlawb-node/src/config.rs | 11 + crates/gitlawb-node/src/db/mod.rs | 145 ++++++++++- crates/gitlawb-node/src/git/mod.rs | 8 +- crates/gitlawb-node/src/git/store.rs | 3 +- crates/gitlawb-node/src/reconciliation.rs | 299 ++++++++++++++-------- 6 files changed, 381 insertions(+), 108 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f3de7570..41aa6ce5 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -213,9 +213,11 @@ pub async fn get_by_cid( /// GET /api/v1/ipfs/pins /// -/// Returns all CIDs that have been pinned to the local IPFS node from git -/// objects received via push. Each entry includes the git SHA-256 hex, the -/// CIDv1 string, and the timestamp when it was pinned. +/// Returns all CIDs that have been pinned from git objects received via push. +/// Each entry includes the git SHA-256 hex, a CIDv1 string, and the timestamp +/// when it was pinned. For Pinata-only rows (no local IPFS pin), the `cid` +/// field carries `pinata_cid` so CLI consumers see a usable value. The raw +/// `pinata_cid` is also surfaced. pub async fn list_pins(State(state): State) -> Result> { let pins = state .db @@ -223,6 +225,21 @@ pub async fn list_pins(State(state): State) -> Result = pins + .into_iter() + .map(|p| { + // Synthesize a usable CID from pinata_cid when this is a + // Pinata-only row (no local IPFS pin). + let display_cid = p.cid.clone().or_else(|| p.pinata_cid.clone()); + serde_json::json!({ + "sha256_hex": p.sha256_hex, + "cid": display_cid, + "pinned_at": p.pinned_at, + "pinata_cid": p.pinata_cid, + }) + }) + .collect(); + Ok(Json(serde_json::json!({ "pins": pins, "count": pins.len(), diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fc2247d9..b892f156 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -86,6 +86,17 @@ pub struct Config { #[arg(long, env = "GITLAWB_AUTO_SYNC", default_value_t = false)] pub auto_sync: bool, + /// Enable the periodic reconciliation sweep that re-derives pin/seal sets + /// and fills durability gaps. Defaults to true; set to false to disable + /// the sweep even when a pin backend (IPFS/Pinata) is configured. + #[arg( + long, + env = "GITLAWB_RECONCILIATION_SWEEP", + default_value_t = true, + action = clap::ArgAction::Set + )] + pub reconciliation_sweep: bool, + /// Irys URL for Arweave permanent anchoring. /// Leave empty to disable. Use https://devnet.irys.xyz for free devnet. #[arg(long, env = "GITLAWB_IRYS_URL", default_value = "")] diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 8f930fd1..cf34993d 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -2422,7 +2422,7 @@ impl Db { .into_iter() .map(|r| PinnedCidRecord { sha256_hex: r.get("sha256_hex"), - cid: r.get("cid"), + cid: r.try_get("cid").ok(), pinned_at: r.get("pinned_at"), pinata_cid: r.get("pinata_cid"), }) @@ -4052,6 +4052,149 @@ mod migration_tests { assert_eq!(attempted_at_of(&db, "z6Mkfoo/failed").await, None); assert_eq!(attempted_at_of(&db, "z6Mkfoo/done").await, None); } + + /// Migration v12 makes pinned_cids.cid nullable so record_pinata_cid can + /// create Pinata-only rows without a local IPFS CID. This test seeds a + /// pre-v12 schema (cid NOT NULL, pinata_cid column exists but no + /// nullability change yet) with rows in each of the three states the + /// has_ipfs_cid / filter_ipfs_pinned_oids predicates must classify: + /// + /// (1) cid IS NOT NULL, pinata_cid IS NULL → has_ipfs = true + /// (2) cid IS NOT NULL, cid != pinata_cid → has_ipfs = true + /// (3) cid IS NOT NULL, cid = pinata_cid (legacy) → has_ipfs = false + /// + /// After the migration we also test that a Pinata-only INSERT (cid = NULL) + /// works and produces has_ipfs = false, has_pinata = true. + #[sqlx::test] + async fn migration_v12_makes_cid_nullable_and_preserves_classification(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + + // Create all tables, then drop the NOT NULL constraint on cid + // and drop schema_migrations records to simulate a pre-v12 node. + db.migrate().await.unwrap(); + sqlx::query("ALTER TABLE pinned_cids ALTER COLUMN cid SET NOT NULL") + .execute(&db.pool) + .await + .unwrap(); + + sqlx::query("DELETE FROM schema_migrations") + .execute(&db.pool) + .await + .unwrap(); + for m in MIGRATIONS.iter().take_while(|m| m.version < 12) { + sqlx::query( + "INSERT INTO schema_migrations (version, name, applied_at) + VALUES ($1, $2, $3)", + ) + .bind(m.version) + .bind(m.name) + .bind("2026-07-01T00:00:00Z") + .execute(&db.pool) + .await + .unwrap(); + } + + // ── Seed legacy rows ─────────────────────────────────────────── + let now = "2026-07-01T12:00:00Z"; + + // (1) Real local IPFS pin, no Pinata. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ($1, $2, $3, $4)", + ) + .bind("sha_real_only") + .bind("QmRealLocalCid") + .bind(now) + .bind(Option::<&str>::None) + .execute(&db.pool) + .await + .unwrap(); + + // (2) Both CIDs present and distinct. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ($1, $2, $3, $4)", + ) + .bind("sha_both_distinct") + .bind("QmLocalForThisBlob") + .bind(now) + .bind("QmPinataForThisBlob") + .execute(&db.pool) + .await + .unwrap(); + + // (3) Legacy row where cid was set to pinata_cid as fallback. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ($1, $2, $3, $4)", + ) + .bind("sha_legacy_fallback") + .bind("QmLegacyEqual") + .bind(now) + .bind("QmLegacyEqual") + .execute(&db.pool) + .await + .unwrap(); + + // ── Apply migration v12 ──────────────────────────────────────── + db.migrate().await.unwrap(); + + // ── Assertions ───────────────────────────────────────────────── + + // Column is now nullable. + let nullable: String = sqlx::query_scalar( + "SELECT is_nullable FROM information_schema.columns + WHERE table_name = 'pinned_cids' AND column_name = 'cid'", + ) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(nullable, "YES", "cid must be nullable after v12"); + + // Classification: has_ipfs_cid. + assert!( + db.has_ipfs_cid("sha_real_only").await.unwrap(), + "real local IPFS CID must be classified as pinned" + ); + assert!( + db.has_ipfs_cid("sha_both_distinct").await.unwrap(), + "distinct local CID must be classified as pinned" + ); + assert!( + !db.has_ipfs_cid("sha_legacy_fallback").await.unwrap(), + "legacy equal-cid row must NOT be classified as having an IPFS CID" + ); + + // has_pinata_cid. + assert!( + !db.has_pinata_cid("sha_real_only").await.unwrap(), + "no pinata_cid means has_pinata = false" + ); + assert!( + db.has_pinata_cid("sha_both_distinct").await.unwrap(), + "non-null pinata_cid means has_pinata = true" + ); + assert!( + db.has_pinata_cid("sha_legacy_fallback").await.unwrap(), + "non-null pinata_cid means has_pinata = true (legacy row)" + ); + + // ── Pinata-only INSERT (new post-v12 row) ────────────────────── + db.record_pinata_cid("sha_pinata_only", "QmPinataOnly") + .await + .unwrap(); + assert!( + !db.has_ipfs_cid("sha_pinata_only").await.unwrap(), + "Pinata-only row must NOT be classified as having a local IPFS CID" + ); + assert!( + db.has_pinata_cid("sha_pinata_only").await.unwrap(), + "Pinata-only row must have has_pinata = true" + ); + + // ── Idempotent re-run ────────────────────────────────────────── + db.migrate().await.unwrap(); + } } #[cfg(test)] diff --git a/crates/gitlawb-node/src/git/mod.rs b/crates/gitlawb-node/src/git/mod.rs index 5dd3dc13..dd6a7283 100644 --- a/crates/gitlawb-node/src/git/mod.rs +++ b/crates/gitlawb-node/src/git/mod.rs @@ -191,7 +191,7 @@ impl GitCommand { // is set while we hold the lock, the sweep cannot drain the // registry until we release it. if let Some(ref ctx) = ctx { - let mut registry = ctx.registry.lock().unwrap(); + let mut registry = ctx.registry.lock().unwrap_or_else(|e| e.into_inner()); if ctx.canceled.load(Ordering::SeqCst) { // Canceled after spawn: kill the whole process group (not // just the immediate child) and wait to avoid zombies. @@ -226,7 +226,11 @@ struct PgidGuard { impl Drop for PgidGuard { fn drop(&mut self) { if let (Some(pgid), Some(ref ctx)) = (self.pgid, &self.ctx) { - ctx.registry.lock().unwrap().remove(&pgid); + let _ = ctx + .registry + .lock() + .unwrap_or_else(|e| e.into_inner()) + .remove(&pgid); } } } diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 229ee695..ceb75531 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -66,9 +66,8 @@ pub fn list_refs(repo_path: &Path) -> Result> { /// Read the current HEAD commit hash of a repository. /// Returns None if the repo is empty (no commits yet). pub fn head_commit(repo_path: &Path) -> Result> { - let output = Command::new("git") + let output = crate::git::GitCommand::new(repo_path) .args(["rev-parse", "--verify", "HEAD"]) - .current_dir(repo_path) .output() .context("failed to run git rev-parse")?; diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs index c2ce5cbc..849b7aa1 100644 --- a/crates/gitlawb-node/src/reconciliation.rs +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::path::PathBuf; use std::sync::atomic::Ordering; use std::sync::Arc; @@ -28,8 +28,23 @@ const MAX_OBJECTS_PER_REPO: usize = 50_000; /// filter). A pathological repo that stalls past this is skipped for the pass. const REPO_SCAN_DEADLINE: Duration = Duration::from_secs(300); +/// Per-repo deadline for the pinning phase (IPFS + Pinata uploads). An +/// unavailable backend that stalls per-object must not hold the sweep for +/// the entire backlog; this bounds the total wall time per repo per pass. +const PIN_PHASE_DEADLINE: Duration = Duration::from_secs(300); + +/// Whether the sweep should spawn given the current configuration. +/// Extracted for testing — test both directions independently. +fn should_spawn(config: &Config) -> bool { + if !config.reconciliation_sweep { + return false; + } + !config.ipfs_api.is_empty() || !config.pinata_jwt.is_empty() +} + /// Spawn the periodic reconciliation sweep background task. -/// No-op when neither IPFS nor Pinata is configured. +/// No-op when neither IPFS nor Pinata is configured, or when +/// `reconciliation_sweep` is disabled. pub fn spawn( db: Arc, config: Arc, @@ -38,8 +53,10 @@ pub fn spawn( node_did: gitlawb_core::did::Did, mut shutdown_rx: watch::Receiver, ) { - if config.ipfs_api.is_empty() && config.pinata_jwt.is_empty() { - tracing::info!("reconciliation sweep: neither IPFS nor Pinata configured, skipping spawn"); + if !should_spawn(&config) { + tracing::info!( + "reconciliation sweep: disabled or neither IPFS nor Pinata configured, skipping spawn" + ); return; } @@ -199,8 +216,14 @@ async fn run_pass( ctx.canceled.store(true, Ordering::SeqCst); #[cfg(unix)] { - let active = ctx.registry.lock().unwrap(); - for &pgid in active.iter() { + let pgids: Vec = ctx + .registry + .lock() + .unwrap_or_else(|e| e.into_inner()) + .iter() + .copied() + .collect(); + for &pgid in &pgids { unsafe { let _ = libc::kill(-pgid, libc::SIGTERM); } @@ -267,35 +290,81 @@ async fn run_pass( } // Visibility may have narrowed mid-scan with a path-scoped deny. - // Recompute the allowed set from fresh rules and intersect it with - // the existing object_list so newly-withheld blobs are excluded from - // the missing sets and never published to a public backend. - let fresh_allowed = crate::git::visibility_pack::replicable_blob_set( - &disk, - &rules, - fresh_repo.is_public, - &fresh_repo.owner_did, - )?; - let object_list: Vec = object_list - .into_iter() - .filter(|oid| fresh_allowed.contains(oid)) - .collect(); + // Recompute the allowed set from fresh rules in a spawn_blocking + // and intersect it with the existing object_list (R1-P1, R1-P2). + let fresh_disk = disk.clone(); + let fresh_rules = rules.clone(); + let fresh_owner = fresh_repo.owner_did.clone(); + let fresh_is_public = fresh_repo.is_public; + let existing_list = object_list; + let refilter_ctx = ctx.clone(); + + let refiltered = tokio::time::timeout( + REPO_SCAN_DEADLINE, + tokio::task::spawn_blocking(move || -> anyhow::Result> { + let _guard = crate::git::set_scan_context(refilter_ctx); + let allowed = crate::git::visibility_pack::replicable_blob_set( + &fresh_disk, + &fresh_rules, + fresh_is_public, + &fresh_owner, + )?; + let all_blobs = crate::git::push_delta::all_blob_oids(&fresh_disk)?; + Ok(crate::git::visibility_pack::replicable_objects_fail_closed( + existing_list, + &allowed, + &all_blobs, + )) + }), + ) + .await; - let ipfs_enabled = !config.ipfs_api.is_empty(); + let object_list: Vec = match refiltered { + Ok(Ok(Ok(list))) => list, + Ok(Ok(Err(e))) => { + tracing::warn!(repo = %repo_slug, err = %e, "fresh-visibility re-filter failed, skipping"); + continue; + } + Ok(Err(e)) => { + tracing::warn!(repo = %repo_slug, err = %e, "fresh-visibility re-filter task panicked, skipping"); + continue; + } + Err(_) => { + ctx.canceled.store(true, Ordering::SeqCst); + #[cfg(unix)] + { + let pgids: Vec = ctx + .registry + .lock() + .unwrap_or_else(|e| e.into_inner()) + .iter() + .copied() + .collect(); + for &pgid in &pgids { + unsafe { + let _ = libc::kill(-pgid, libc::SIGTERM); + } + } + } + tracing::warn!(repo = %repo_slug, "fresh-visibility re-filter deadline exceeded, skipped"); + continue; + } + }; + + let _ipfs_enabled = !config.ipfs_api.is_empty(); let pinata_enabled = !config.pinata_jwt.is_empty(); - // Only compute missing sets for enabled backends so that a Pinata-only - // or IPFS-only node does not report permanent unfillable gaps. - let ipfs_missing: Vec = if ipfs_enabled { - let already = match db.filter_ipfs_pinned_oids(&object_list).await { - Ok(v) => v, - Err(e) => { - tracing::warn!(repo = %repo_slug, err = %e, "filter_ipfs_pinned_oids failed, skipping"); - continue; - } - }; + // IPFS-missing set (capped). + let already_ipfs = match db.filter_ipfs_pinned_oids(&object_list).await { + Ok(v) => v, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "filter_ipfs_pinned_oids failed, skipping"); + continue; + } + }; + let ipfs_missing: Vec = { let all_set: HashSet<&str> = object_list.iter().map(|s| s.as_str()).collect(); - let done_set: HashSet<&str> = already.iter().map(|s| s.as_str()).collect(); + let done_set: HashSet<&str> = already_ipfs.iter().map(|s| s.as_str()).collect(); let mut v: Vec = all_set .difference(&done_set) .map(|s| s.to_string()) @@ -309,8 +378,6 @@ async fn run_pass( ); } v - } else { - Vec::new() }; let pinata_missing: Vec = if pinata_enabled { @@ -348,13 +415,21 @@ async fn run_pass( crate::metrics::record_reconciliation_gaps_found(repo_gaps as u64); } - let pinned_ipfs = if ipfs_enabled && !ipfs_missing.is_empty() { - crate::ipfs_pin::pin_new_objects(&config.ipfs_api, &disk, ipfs_missing, db).await - } else { - Vec::new() + let pinned_ipfs = match tokio::time::timeout( + PIN_PHASE_DEADLINE, + crate::ipfs_pin::pin_new_objects(&config.ipfs_api, &disk, ipfs_missing, db), + ) + .await + { + Ok(v) => v, + Err(_) => { + tracing::warn!(repo = %repo_slug, "IPFS pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + Vec::new() + } }; - let pinned_pinata = if pinata_enabled && !pinata_missing.is_empty() { + let pinned_pinata = match tokio::time::timeout( + PIN_PHASE_DEADLINE, crate::pinata::pin_new_objects( http_client, &config.pinata_upload_url, @@ -362,10 +437,15 @@ async fn run_pass( &disk, pinata_missing, db, - ) - .await - } else { - Vec::new() + ), + ) + .await + { + Ok(v) => v, + Err(_) => { + tracing::warn!(repo = %repo_slug, "Pinata pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + Vec::new() + } }; let repo_filled = pinned_ipfs.len() + pinned_pinata.len(); @@ -466,8 +546,14 @@ async fn run_pass( ctx2.canceled.store(true, Ordering::SeqCst); #[cfg(unix)] { - let active = ctx2.registry.lock().unwrap(); - for &pgid in active.iter() { + let pgids: Vec = ctx2 + .registry + .lock() + .unwrap_or_else(|e| e.into_inner()) + .iter() + .copied() + .collect(); + for &pgid in &pgids { unsafe { let _ = libc::kill(-pgid, libc::SIGTERM); } @@ -496,52 +582,30 @@ async fn run_pass( // This avoids unbounded Irys writes on a timer — repos // with no withheld changes do not re-anchor the manifest. if !sealed.is_empty() && !config.irys_url.is_empty() { - let all_existing = match db.list_all_encrypted_blobs(&repo.id).await { - Ok(v) => v, - Err(e) => { - tracing::warn!( - repo = %repo_slug, - err = %e, - "list_all_encrypted_blobs failed, skipping anchor" - ); - continue; - } + let owner_short = crate::db::normalize_owner_key(&repo.owner_did); + let slug = format!("{}/{}", owner_short, repo.name); + let ts = chrono::Utc::now().to_rfc3339(); + let node_did_str = node_did.to_string(); + + let manifest = crate::arweave::EncryptedManifest { + repo: &slug, + owner_did: &repo.owner_did, + node_did: &node_did_str, + timestamp: &ts, + blobs: &sealed, }; - if !all_existing.is_empty() { - let owner_short = crate::db::normalize_owner_key(&repo.owner_did); - let slug = format!("{}/{}", owner_short, repo.name); - let ts = chrono::Utc::now().to_rfc3339(); - let node_did_str = node_did.to_string(); - - let mut blob_map: HashMap = HashMap::new(); - for (oid, cid) in &all_existing { - blob_map.insert(oid.clone(), cid.clone()); - } - for (oid, cid) in &sealed { - blob_map.insert(oid.clone(), cid.clone()); - } - let merged: Vec<(String, String)> = blob_map.into_iter().collect(); - - let manifest = crate::arweave::EncryptedManifest { - repo: &slug, - owner_did: &repo.owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &merged, - }; - if let Err(e) = crate::arweave::anchor_encrypted_manifest( - http_client, - &config.irys_url, - &manifest, - ) - .await - { - tracing::warn!( - repo = %slug, - err = %e, - "encrypted manifest anchor failed (will retry next pass)" - ); - } + if let Err(e) = crate::arweave::anchor_encrypted_manifest( + http_client, + &config.irys_url, + &manifest, + ) + .await + { + tracing::warn!( + repo = %slug, + err = %e, + "encrypted manifest anchor failed (will retry next pass)" + ); } } } @@ -564,25 +628,61 @@ mod tests { std::sync::Arc::new(cfg) } + /// Build a config with IPFS API set so the gate fires the other way. + fn ipfs_config() -> std::sync::Arc { + let cfg = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + "http://127.0.0.1:5001", + ]); + std::sync::Arc::new(cfg) + } + + #[test] + fn should_spawn_false_when_both_empty() { + let cfg = empty_pin_config(); + assert!(!super::should_spawn(&cfg)); + } + + #[test] + fn should_spawn_true_when_ipfs_set() { + let cfg = ipfs_config(); + assert!(super::should_spawn(&cfg)); + } + + #[test] + fn should_spawn_true_when_pinata_set() { + let cfg = ::parse_from([ + "gitlawb-node-test", + "--pinata-jwt", + "test-jwt", + ]); + assert!(super::should_spawn(&cfg)); + } + + #[test] + fn should_spawn_false_when_sweep_disabled() { + let cfg = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + "http://127.0.0.1:5001", + "--reconciliation-sweep", + "false", + ]); + assert!(!super::should_spawn(&cfg)); + } + /// spawn() must return immediately (without panicking or touching the DB) /// when neither IPFS nor Pinata is configured. This proves the gate - /// branch at the top of spawn() is actually reachable: if the gate were - /// deleted or the field names changed, spawn() would call tokio::spawn - /// and then hit a missing-DB panic on the first pass instead of - /// returning, causing the test to time out or panic. + /// branch at the top of spawn() is actually reachable. #[tokio::test] async fn test_spawn_gate_skips_when_no_pin_backends_configured() { let config = empty_pin_config(); - // Sanity: the config we built really has empty pin fields. assert!(config.ipfs_api.is_empty(), "ipfs_api should be empty"); assert!(config.pinata_jwt.is_empty(), "pinata_jwt should be empty"); - // We cannot construct a real Db without Postgres, but spawn() must - // return before using the Db when both pin backends are disabled. // Use a dummy Db built from a disconnected pool; spawn() must not // reach any code that would touch it. - // max_connections(1): crossbeam-queue requires capacity >= 1; - // the pool is connect_lazy with a bogus URL so no connection is made. let pool = sqlx::postgres::PgPoolOptions::new() .max_connections(1) .connect_lazy("postgresql://localhost/gitlawb_test_nonexistent") @@ -598,8 +698,7 @@ mod tests { super::spawn(db, config, http, kp, node_did, rx); } - /// Constant smoke-check kept as a compile-time tripwire. The real gate - /// behaviour is covered by test_spawn_gate_skips_when_no_pin_backends_configured. + /// Constant smoke-check kept as a compile-time tripwire. #[test] fn sweep_interval_constant_is_nonzero() { assert_ne!(super::SWEEP_INTERVAL_SECS, 0); From 9dc9bd6b69fe0450a0059acfe68703f952d58a6c Mon Sep 17 00:00:00 2001 From: Gravirei Date: Fri, 7 Aug 2026 17:22:51 +0600 Subject: [PATCH 10/26] fix(db): add node_state table and clean up pinned_cid provenance - Add migration v18 clearing legacy rows where cid was backfilled from pinata_cid, so provenance is recorded instead of inferred from CID inequality (R2-P2) - Add migration v19 creating the node_state key/value table for the sweep's keyset cursor (R2-P1), plus get/set helpers - record_pinned_cid now unconditionally overwrites a stale wrong CID so the sweep can repair objects pinned with bad bytes (R1-P2) - has_ipfs_cid / filter_ipfs_pinned_oids reduce to cid IS NOT NULL - Chunk filter_ipfs_pinned_oids / filter_pinata_pinned_oids to bound ANY($1) array size on uncapped object lists (R1-P3) - Add tests for the v18/v19 migrations, node_state roundtrip, and the writer repair behavior --- crates/gitlawb-node/src/db/mod.rs | 320 +++++++++++++++++++++++++++--- 1 file changed, 289 insertions(+), 31 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index cf34993d..278bcec3 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -916,6 +916,43 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE sync_queue ADD COLUMN IF NOT EXISTS attempted_at TEXT", ], }, + // 18 clears the range #135/#173 claimed (13/14) and #253 (16), so the + // backfill below cannot collide with any branch in flight. 18/19 are the + // next free integers after the merged max (17). + Migration { + version: 18, + name: "pinned_cids_clear_legacy_equal_cid", + stmts: &[ + // R2-P2 provenance fix: v12 allowed cid = pinata_cid as a fallback for + // new rows, which made has_ipfs_cid infer local-IPFS provenance from + // CID *inequality*. That inference is wrong for a mirror whose + // Pinata CID happens to equal the local CID. Clear the legacy + // equal-cid rows so `has_ipfs_cid` reduces to `cid IS NOT NULL` and + // provenance is recorded, never guessed. The sweep re-pins the now + // "missing" objects on its next pass (they get a real cid then). + r#"UPDATE pinned_cids + SET cid = NULL + WHERE cid IS NOT NULL + AND pinata_cid IS NOT NULL + AND cid = pinata_cid"#, + ], + }, + Migration { + version: 19, + name: "node_state", + stmts: &[ + // R2-P1 cursor persistence: the reconciliation sweep's keyset cursor + // must survive a node restart so a dropped pass resumes where it + // stopped instead of re-walking every repo. A tiny key/value table + // keyed by opaque string (never a column per feature) keeps this a + // generic primitive future features can reuse. + r#"CREATE TABLE IF NOT EXISTS node_state ( + key TEXT NOT NULL PRIMARY KEY, + value TEXT NOT NULL, + updated_at TEXT NOT NULL + )"#, + ], + }, ]; // ── Repos ───────────────────────────────────────────────────────────────────── @@ -2318,21 +2355,62 @@ impl Db { } } +// ── Node state ──────────────────────────────────────────────────────────────── + +impl Db { + /// Read an opaque node-state value. Returns `None` when the key has never + /// been written. Used by the reconciliation sweep to persist its keyset + /// cursor across restarts (R2-P1). + pub async fn get_node_state(&self, key: &str) -> Result> { + let row = sqlx::query("SELECT value FROM node_state WHERE key = $1") + .bind(key) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.get("value"))) + } + + /// Write an opaque node-state value (upsert). `None` deletes the key so a + /// cleared cursor does not accumulate stale rows. + pub async fn set_node_state(&self, key: &str, value: Option<&str>) -> Result<()> { + match value { + Some(v) => { + sqlx::query( + "INSERT INTO node_state (key, value, updated_at) + VALUES ($1, $2, $3) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = EXCLUDED.updated_at", + ) + .bind(key) + .bind(v) + .bind(Utc::now().to_rfc3339()) + .execute(&self.pool) + .await?; + } + None => { + sqlx::query("DELETE FROM node_state WHERE key = $1") + .bind(key) + .execute(&self.pool) + .await?; + } + } + Ok(()) + } +} + // ── Pinned CIDs ─────────────────────────────────────────────────────────────── impl Db { /// Record the local IPFS CID for a git object. - /// If a Pinata-only row already exists (cid IS NULL or was set as Pinata - /// fallback so cid = pinata_cid), this replaces it with the real IPFS CID. + /// This unconditionally replaces any prior `cid` (NULL, Pinata-only, a + /// legacy fallback where cid = pinata_cid, OR a stale wrong CID). A stale + /// local CID must be repairable by the sweep, otherwise an object pinned + /// once with the wrong bytes is never corrected (R1-P2). pub async fn record_pinned_cid(&self, sha256_hex: &str, cid: &str) -> Result<()> { sqlx::query( "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3) ON CONFLICT(sha256_hex) DO UPDATE SET cid = EXCLUDED.cid, - pinned_at = EXCLUDED.pinned_at - WHERE pinned_cids.cid IS NULL - OR pinned_cids.cid = pinned_cids.pinata_cid", + pinned_at = EXCLUDED.pinned_at", ) .bind(sha256_hex) .bind(cid) @@ -2429,14 +2507,14 @@ impl Db { .collect()) } - /// Returns true when this object has a real local IPFS CID (not a legacy - /// Pinata fallback where cid was set to pinata_cid for new rows). + /// Returns true when this object has a real local IPFS CID. After migration + /// v18 cleared legacy `cid = pinata_cid` fallback rows (provenance is now + /// recorded, never inferred), `cid IS NOT NULL` is the complete predicate. pub async fn has_ipfs_cid(&self, sha256_hex: &str) -> Result { let row = sqlx::query( "SELECT COUNT(*) as cnt FROM pinned_cids WHERE sha256_hex = $1 - AND cid IS NOT NULL - AND cid IS DISTINCT FROM pinata_cid", + AND cid IS NOT NULL", ) .bind(sha256_hex) .fetch_one(&self.pool) @@ -2457,48 +2535,72 @@ impl Db { /// Given a list of sha256_hex values, returns the subset that already have /// a Pinata CID recorded. Used by the reconciliation sweep to skip objects - /// that Pinata has already handled. + /// that Pinata has already handled. Chunked like `filter_ipfs_pinned_oids` + /// to bound the `ANY($1)` array size on full uncapped object lists (R1-P3). pub async fn filter_pinata_pinned_oids(&self, oids: &[String]) -> Result> { if oids.is_empty() { return Ok(Vec::new()); } - let rows = sqlx::query( - "SELECT sha256_hex FROM pinned_cids WHERE sha256_hex = ANY($1) AND pinata_cid IS NOT NULL", - ) - .bind(oids) - .fetch_all(&self.pool) - .await?; - Ok(rows.into_iter().map(|r| r.get("sha256_hex")).collect()) + const CHUNK_SIZE: usize = 1000; + let mut out = Vec::new(); + for chunk in oids.chunks(CHUNK_SIZE) { + let rows = sqlx::query( + "SELECT sha256_hex FROM pinned_cids WHERE sha256_hex = ANY($1) AND pinata_cid IS NOT NULL", + ) + .bind(chunk) + .fetch_all(&self.pool) + .await?; + out.extend(rows.into_iter().map(|r| r.get("sha256_hex"))); + } + Ok(out) } /// Given a list of sha256_hex values, returns the subset that have a real - /// local IPFS CID (excluding legacy Pinata fallback rows where cid equals - /// pinata_cid). Used by the reconciliation sweep to skip IPFS-complete objects. + /// local IPFS CID (`cid IS NOT NULL`; after migration v18 provenance is + /// recorded, never inferred from CID inequality). Used by the reconciliation + /// sweep to skip IPFS-complete objects. + /// + /// The input is processed in fixed-size chunks so the `ANY($1)` array sent + /// to Postgres is bounded even when the sweep hands over a full uncapped + /// object list (R1-P3). pub async fn filter_ipfs_pinned_oids(&self, oids: &[String]) -> Result> { if oids.is_empty() { return Ok(Vec::new()); } - let rows = sqlx::query( - "SELECT sha256_hex FROM pinned_cids - WHERE sha256_hex = ANY($1) - AND cid IS NOT NULL - AND cid IS DISTINCT FROM pinata_cid", - ) - .bind(oids) - .fetch_all(&self.pool) - .await?; - Ok(rows.into_iter().map(|r| r.get("sha256_hex")).collect()) + const CHUNK_SIZE: usize = 1000; + let mut out = Vec::new(); + for chunk in oids.chunks(CHUNK_SIZE) { + let rows = sqlx::query( + "SELECT sha256_hex FROM pinned_cids + WHERE sha256_hex = ANY($1) + AND cid IS NOT NULL", + ) + .bind(chunk) + .fetch_all(&self.pool) + .await?; + out.extend(rows.into_iter().map(|r| r.get("sha256_hex"))); + } + Ok(out) } /// Record the Pinata CID for a git object. /// `cid` is left NULL for new rows so that `has_ipfs_cid` (which checks /// `cid IS NOT NULL`) correctly distinguishes local IPFS state from - /// Pinata-only state. + /// Pinata-only state. A legacy row holding `cid = pinata_cid` fallback is + /// cleared here as well (belt-and-suspenders alongside migration v18): the + /// Pinata upload proves nothing about local IPFS state, so the old fallback + /// value must not be misread as a local pin. pub async fn record_pinata_cid(&self, sha256_hex: &str, pinata_cid: &str) -> Result<()> { sqlx::query( "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) VALUES ($1, $2, $3, $4) - ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid", + ON CONFLICT(sha256_hex) DO UPDATE SET + pinata_cid = EXCLUDED.pinata_cid, + cid = CASE + WHEN pinned_cids.cid IS NOT NULL + AND pinned_cids.cid = pinned_cids.pinata_cid THEN NULL + ELSE pinned_cids.cid + END", ) .bind(sha256_hex) .bind(Option::<&str>::None) // cid is NULL for Pinata-only new rows @@ -4063,6 +4165,10 @@ mod migration_tests { /// (2) cid IS NOT NULL, cid != pinata_cid → has_ipfs = true /// (3) cid IS NOT NULL, cid = pinata_cid (legacy) → has_ipfs = false /// + /// Legacy row (3) stops being a special case because migration v18 clears + /// `cid = pinata_cid` back to NULL, so `has_ipfs_cid` reduces to the plain + /// `cid IS NOT NULL` predicate (provenance recorded, never inferred). + /// /// After the migration we also test that a Pinata-only INSERT (cid = NULL) /// works and produces has_ipfs = false, has_pinata = true. #[sqlx::test] @@ -4195,6 +4301,158 @@ mod migration_tests { // ── Idempotent re-run ────────────────────────────────────────── db.migrate().await.unwrap(); } + + /// Migration v18 clears legacy rows where cid was set to pinata_cid as a + /// fallback, so `has_ipfs_cid` no longer has to infer provenance from CID + /// inequality (R2-P2). Rows where the CIDs genuinely differ are untouched. + #[sqlx::test] + async fn migration_v18_clears_legacy_equal_cid(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + let now = "2026-07-01T12:00:00Z"; + + // Seed one legacy equal-cid row and one distinct-cid row, then mark + // v18 (and v19, applied after it) as not yet run so re-running + // migrate() exercises the backfill in isolation. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ('sha_equal', 'QmSame', $1, 'QmSame'), + ('sha_distinct', 'QmLocal', $1, 'QmPinata')", + ) + .bind(now) + .execute(&db.pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version >= 18") + .execute(&db.pool) + .await + .unwrap(); + + db.migrate().await.unwrap(); + + // Backfilled row now has no local CID; distinct row is untouched. + assert!( + !db.has_ipfs_cid("sha_equal").await.unwrap(), + "legacy equal-cid row must be cleared to NULL by v18" + ); + assert!( + db.has_ipfs_cid("sha_distinct").await.unwrap(), + "distinct-cid row must survive the backfill" + ); + assert!(db.has_pinata_cid("sha_equal").await.unwrap()); + } + + /// Migration v19 creates the node_state key/value table and the get/set + /// helpers round-trip through it (used by the sweep cursor persistence). + #[sqlx::test] + async fn node_state_roundtrip_and_delete(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + + assert_eq!( + db.get_node_state("sweep_cursor").await.unwrap(), + None, + "absent key reads as None" + ); + + db.set_node_state("sweep_cursor", Some("repo/b")) + .await + .unwrap(); + assert_eq!( + db.get_node_state("sweep_cursor").await.unwrap(), + Some("repo/b".to_string()), + "value survives a write + read" + ); + + // Upsert overwrites. + db.set_node_state("sweep_cursor", Some("repo/c")) + .await + .unwrap(); + assert_eq!( + db.get_node_state("sweep_cursor").await.unwrap(), + Some("repo/c".to_string()) + ); + + // None deletes the key. + db.set_node_state("sweep_cursor", None).await.unwrap(); + assert_eq!(db.get_node_state("sweep_cursor").await.unwrap(), None); + } + + /// record_pinned_cid must repair a stale WRONG local CID, not only fill a + /// NULL or Pinata-fallback slot (R1-P2): an object pinned once with the + /// wrong bytes is re-pinned by the sweep and the row overwritten. + #[sqlx::test] + async fn record_pinned_cid_repairs_stale_wrong_cid(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + + // A stale wrong CID that is neither NULL nor equal to pinata_cid. + db.record_pinned_cid("sha_stale", "QmStaleWrong") + .await + .unwrap(); + db.record_pinata_cid("sha_stale", "QmPinataX") + .await + .unwrap(); + + // Re-pin with the correct CID — must overwrite despite the existing + // distinct cid column. + db.record_pinned_cid("sha_stale", "QmCorrect") + .await + .unwrap(); + + let cid: String = + sqlx::query_scalar("SELECT cid FROM pinned_cids WHERE sha256_hex = 'sha_stale'") + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(cid, "QmCorrect", "stale wrong CID must be repaired"); + } + + /// record_pinata_cid must clear a legacy cid = pinata_cid fallback (v18's + /// belt-and-suspenders) so a later Pinata-only row is never misread as a + /// local IPFS pin. + #[sqlx::test] + async fn record_pinata_cid_clears_legacy_equal_cid(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + + db.record_pinned_cid("sha_fallback", "QmFallback") + .await + .unwrap(); + // Simulate a legacy row where cid was forced equal to pinata_cid. + sqlx::query( + "UPDATE pinned_cids SET pinata_cid = 'QmFallback' WHERE sha256_hex = 'sha_fallback'", + ) + .execute(&db.pool) + .await + .unwrap(); + + // Recording a new (different) Pinata CID must NULL the stale fallback cid. + db.record_pinata_cid("sha_fallback", "QmPinataNew") + .await + .unwrap(); + + let cid: Option = + sqlx::query_scalar("SELECT cid FROM pinned_cids WHERE sha256_hex = 'sha_fallback'") + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(cid, None, "legacy equal-cid fallback must be cleared"); + + // But a genuine local pin plus a distinct Pinata CID is preserved. + db.record_pinned_cid("sha_genuine", "QmLocalGenuine") + .await + .unwrap(); + db.record_pinata_cid("sha_genuine", "QmPinataGenuine") + .await + .unwrap(); + let cid: String = + sqlx::query_scalar("SELECT cid FROM pinned_cids WHERE sha256_hex = 'sha_genuine'") + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(cid, "QmLocalGenuine"); + } } #[cfg(test)] From 7db67549045c03ad41f716525eb1c2c05e6d27b3 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Fri, 7 Aug 2026 17:22:56 +0600 Subject: [PATCH 11/26] fix(reconciliation): gate spawn on config, persist cursor, and harden the sweep - spawn() returns bool and skips work when the sweep is disabled - Persist the keyset cursor in node_state and only advance it after a batch completes, so an interrupted pass resumes where it stopped (R2-P1) - Skip mirror rows (slash-form id): they replicate with no visibility rules, so the sweep gate is vacuous for them - Re-check quarantine and visibility against fresh rules before each backend pin (R2-P3) - Re-derive the allowed public-object set from fresh rules after the git scan and intersect it with the scanned list before pinning - Move the repos_scanned counter after mirror-row and missing-disk skips - Count only DB-persisted pins as filled and return the real processed count - Escalate SIGTERM to SIGKILL after KILL_GRACE_SECS for wedged git subprocesses - Time out the encrypt_and_pin and pin phases - Deterministic OID ordering and unique gap counting via a union set - Isolate per-backend filter errors and add integration tests for gap repair, mirror skip, and cursor persistence --- crates/gitlawb-node/src/reconciliation.rs | 924 +++++++++++++++------- 1 file changed, 650 insertions(+), 274 deletions(-) diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs index 849b7aa1..7cd646c0 100644 --- a/crates/gitlawb-node/src/reconciliation.rs +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -33,6 +33,15 @@ const REPO_SCAN_DEADLINE: Duration = Duration::from_secs(300); /// the entire backlog; this bounds the total wall time per repo per pass. const PIN_PHASE_DEADLINE: Duration = Duration::from_secs(300); +/// Grace period between the SIGTERM sweep of a stalled repo's git subprocesses +/// and the SIGKILL escalation. `git` normally exits promptly on TERM; only a +/// wedged process should survive this long. +const KILL_GRACE_SECS: u64 = 10; + +/// node_state key under which the sweep's keyset cursor is persisted across +/// restarts (R2-P1). +const CURSOR_KEY: &str = "reconciliation_sweep_cursor"; + /// Whether the sweep should spawn given the current configuration. /// Extracted for testing — test both directions independently. fn should_spawn(config: &Config) -> bool { @@ -44,7 +53,8 @@ fn should_spawn(config: &Config) -> bool { /// Spawn the periodic reconciliation sweep background task. /// No-op when neither IPFS nor Pinata is configured, or when -/// `reconciliation_sweep` is disabled. +/// `reconciliation_sweep` is disabled. Returns `true` when the worker was +/// actually spawned so the caller can gate its own "worker started" logging. pub fn spawn( db: Arc, config: Arc, @@ -52,17 +62,26 @@ pub fn spawn( node_keypair: Arc, node_did: gitlawb_core::did::Did, mut shutdown_rx: watch::Receiver, -) { +) -> bool { if !should_spawn(&config) { tracing::info!( "reconciliation sweep: disabled or neither IPFS nor Pinata configured, skipping spawn" ); - return; + return false; } tokio::spawn(async move { let node_seed = *node_keypair.to_seed(); - let mut cursor: Option = None; + // Resume from the persisted cursor (R2-P1): a node restart must not + // re-walk every repo, and the cursor is only ever advanced after a + // batch completes, so an interrupted pass resumes where it stopped. + let mut cursor: Option = match db.get_node_state(CURSOR_KEY).await { + Ok(v) => v, + Err(e) => { + tracing::warn!(err = %e, "failed to load reconciliation sweep cursor from node_state; starting from scratch"); + None + } + }; loop { let start = std::time::Instant::now(); @@ -107,9 +126,193 @@ pub fn spawn( } } }); + + true +} + +/// Re-derive the *allowed* public-object set from fresh rules and intersect it +/// with the scanned object list. Returns `None` when the re-derivation failed +/// (caller skips the repo). This is the path-scoped-visibility re-filter that +/// runs against rules re-fetched after the git scan, so a narrowing made +/// mid-scan is honored before anything is pinned. +/// +/// The two `spawn_blocking` stages (full scan, re-filter) share the deadline so +/// the total blocking time per repo stays bounded. +async fn refilter_public_objects( + ctx: &Arc, + disk: &std::path::Path, + rules: &[crate::db::VisibilityRule], + is_public: bool, + owner_did: &str, + object_list: Vec, +) -> Option> { + let ctx_clone = ctx.clone(); + let disk_clone = disk.to_path_buf(); + let rules_clone = rules.to_vec(); + let owner_clone = owner_did.to_string(); + + match tokio::time::timeout( + REPO_SCAN_DEADLINE, + tokio::task::spawn_blocking(move || -> anyhow::Result> { + let _guard = crate::git::set_scan_context(ctx_clone.clone()); + let allowed = crate::git::visibility_pack::replicable_blob_set( + &disk_clone, + &rules_clone, + is_public, + &owner_clone, + )?; + if ctx_clone.canceled.load(Ordering::SeqCst) { + return Err(anyhow::anyhow!("scan canceled after replicable_blob_set")); + } + let all_blobs = crate::git::push_delta::all_blob_oids(&disk_clone)?; + Ok(crate::git::visibility_pack::replicable_objects_fail_closed( + object_list, + &allowed, + &all_blobs, + )) + }), + ) + .await + { + Ok(Ok(Ok(list))) => Some(list), + Ok(Ok(Err(e))) => { + tracing::warn!(err = %e, "visibility re-derivation failed"); + None + } + Ok(Err(e)) => { + tracing::warn!(err = %e, "visibility re-derivation task panicked"); + None + } + Err(_) => { + escalate_kill(ctx, "visibility re-derivation deadline exceeded"); + None + } + } +} +/// Re-check quarantine AND root visibility immediately before an irreversible +/// public pin (R1-P1). Returns the fresh repo row plus fresh rules, or `None` +/// when the pin must be skipped. DB failures are treated as skip (never pin on +/// a stale allow), so one repo's failure does not abort the pass. +async fn recheck_public_pin( + db: &Db, + repo_id: &str, + repo_slug: &str, +) -> Option<(crate::db::RepoRecord, Vec)> { + match db.is_repo_quarantined(repo_id).await { + Ok(true) => { + tracing::warn!(repo = %repo_slug, "repo quarantined, skipping pin"); + return None; + } + Ok(false) => {} + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "quarantine recheck failed, skipping pin"); + return None; + } + } + let rules = match db.list_visibility_rules(repo_id).await { + Ok(r) => r, + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "visibility rules re-fetch failed, skipping pin"); + return None; + } + }; + let fresh = match db.get_repo_by_id(repo_id).await { + Ok(Some(r)) => r, + Ok(None) => { + tracing::warn!(repo = %repo_slug, "repo disappeared from DB, skipping pin"); + return None; + } + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "repo re-fetch failed, skipping pin"); + return None; + } + }; + if !crate::visibility::listable_at_root(&rules, fresh.is_public, &fresh.owner_did, None) { + tracing::warn!(repo = %repo_slug, "visibility narrowed, skipping pin"); + return None; + } + Some((fresh, rules)) +} + +/// SIGTERM every registered git process group; after `KILL_GRACE_SECS` re-scan +/// the registry and SIGKILL anything still alive. The escalation task is +/// fire-and-forget: a git process that ignores TERM must not be left running. +fn escalate_kill(ctx: &Arc, reason: &str) { + tracing::warn!(reason, "killing active git subprocesses"); + #[cfg(unix)] + { + let pgids: Vec = ctx + .registry + .lock() + .unwrap_or_else(|e| e.into_inner()) + .iter() + .copied() + .collect(); + for &pgid in &pgids { + unsafe { + let _ = libc::kill(-pgid, libc::SIGTERM); + } + } + } + + let ctx = ctx.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(KILL_GRACE_SECS)).await; + #[cfg(unix)] + { + let pgids: Vec = ctx + .registry + .lock() + .unwrap_or_else(|e| e.into_inner()) + .iter() + .copied() + .collect(); + for &pgid in &pgids { + unsafe { + let _ = libc::kill(-pgid, libc::SIGKILL); + } + } + } + }); +} + +/// Compute the deterministic missing set: `all` minus `done`, sorted so two +/// passes over the same data yield the same pin order. Not capped here — the +/// caller applies the cap and logs a truncation warning. +fn missing_oids(all: &[String], done: &[String]) -> Vec { + let done_set: HashSet<&str> = done.iter().map(|s| s.as_str()).collect(); + let mut missing: Vec = all + .iter() + .filter(|s| !done_set.contains(s.as_str())) + .cloned() + .collect(); + missing.sort(); + missing +} + +/// Cap a missing set, logging once when it was truncated. +fn cap_missing(v: Vec, repo_slug: &str, backend: &str) -> Vec { + if v.len() > MAX_OBJECTS_PER_REPO { + tracing::warn!( + repo = %repo_slug, + backend, + cap = MAX_OBJECTS_PER_REPO, + "per-repo missing cap reached, truncating" + ); + let mut v = v; + v.truncate(MAX_OBJECTS_PER_REPO); + v + } else { + v + } } /// Run one sweep pass. Returns `(repos_scanned, gaps_found, gaps_filled)`. +/// +/// `repos_scanned` counts every repo actually visited this pass (mirror rows +/// and hard skips excluded, and the loop stops counting the moment a shutdown +/// signal breaks the batch), so the returned value never overreports work that +/// a mid-pass shutdown prevented (R1-P3). async fn run_pass( db: &Db, config: &Config, @@ -128,18 +331,28 @@ async fn run_pass( .await?; if batch.is_empty() { + // Covered everything: clear the persisted cursor so the next pass + // starts a fresh cycle instead of wedging on a stale key. *cursor = None; + db.set_node_state(CURSOR_KEY, None).await?; return Ok((0, 0, 0)); } - *cursor = Some(batch.last().unwrap().id.clone()); + // Advance the in-memory cursor now so the next page in this run continues + // after this batch; the PERSISTED cursor is only moved once the batch fully + // completes below, so an interrupted batch is re-walked on restart. + let batch_last = batch.last().unwrap().id.clone(); + *cursor = Some(batch_last.clone()); let mut total_gaps_found = 0usize; let mut total_gaps_filled = 0usize; + let mut repos_scanned = 0usize; + let mut batch_completed = true; for repo in &batch { if *shutdown_rx.borrow() { tracing::info!("reconciliation sweep: shutdown signal received mid-pass, exiting"); + batch_completed = false; break; } @@ -149,12 +362,41 @@ async fn run_pass( repo.name ); + // Mirror rows carry a slash-form id written only by upsert_mirror_repo; + // they hardcode is_public = true and replicate no visibility rules, so a + // sweep over one would irreversibly publish content that the canonical + // gate never admitted (R2-P1). Skip them — the canonical row (if any) + // is swept under its own id. + if repo.id.contains('/') { + tracing::debug!(repo = %repo_slug, "mirror row (no canonical repo), skipping sweep"); + continue; + } + let disk = PathBuf::from(&repo.disk_path); if !disk.exists() { tracing::warn!(repo = %repo_slug, "disk path missing, skipping"); continue; } + // Counted only once the repo has a real chance of work: mirror rows and + // missing-disk rows are hard skips and never count as scanned (R1-P3). + repos_scanned += 1; + + // Cheap quarantine pre-check BEFORE the expensive git scan (R1-P3): + // a repo quarantined since admission should not burn a full scan just + // to be told to skip. + match db.is_repo_quarantined(&repo.id).await { + Ok(true) => { + tracing::warn!(repo = %repo_slug, "repo quarantined, skipping"); + continue; + } + Ok(false) => {} + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "quarantine check failed, skipping"); + continue; + } + } + let rules = match db.list_visibility_rules(&repo.id).await { Ok(r) => r, Err(e) => { @@ -167,16 +409,14 @@ async fn run_pass( continue; } - // Bound the blocking git scan with a deadline so a pathological repo - // cannot stall the entire pass. + // ── Full git scan (bounded) ───────────────────────────────────── + let ctx = crate::git::ScanContext::new(); + let ctx_clone = ctx.clone(); let disk_clone = disk.clone(); let owner_clone = repo.owner_did.clone(); let rules_clone = rules.clone(); let is_public = repo.is_public; - let ctx = crate::git::ScanContext::new(); - let ctx_clone = ctx.clone(); - let object_list = tokio::time::timeout( REPO_SCAN_DEADLINE, tokio::task::spawn_blocking(move || -> anyhow::Result> { @@ -213,22 +453,7 @@ async fn run_pass( continue; } Err(_) => { - ctx.canceled.store(true, Ordering::SeqCst); - #[cfg(unix)] - { - let pgids: Vec = ctx - .registry - .lock() - .unwrap_or_else(|e| e.into_inner()) - .iter() - .copied() - .collect(); - for &pgid in &pgids { - unsafe { - let _ = libc::kill(-pgid, libc::SIGTERM); - } - } - } + escalate_kill(&ctx, "full-scan deadline exceeded"); tracing::warn!(repo = %repo_slug, "full-scan deadline exceeded, killed active git subprocesses, skipping"); continue; } @@ -239,215 +464,133 @@ async fn run_pass( } // ── Phase 1: Public-object pinning (IPFS + Pinata) ──────────────── - // Compute the actually-missing set per backend from the FULL object - // list (no pre-cap) so trailing objects are never excluded. The cap - // applies to the missing sets, bounding pin work. - // - // Recheck quarantine AND visibility before pinning. Rules and - // is_public were fetched once before the scan and may have narrowed - // since; for content-addressed public pins a stale allow is - // effectively irreversible. - match db.is_repo_quarantined(&repo.id).await { - Ok(true) => { - tracing::warn!(repo = %repo_slug, "repo quarantined, skipping"); - continue; - } - Ok(false) => {} - Err(e) => { - tracing::warn!(repo = %repo_slug, err = %e, "quarantine check failed, skipping"); - continue; - } - } - // Recheck visibility rules (P2): owner may have narrowed visibility - // mid-scan. Re-fetch from DB rather than relying on the snapshot - // taken before the git walk. - let rules = match db.list_visibility_rules(&repo.id).await { - Ok(r) => r, - Err(e) => { - tracing::warn!(repo = %repo_slug, err = %e, "visibility rules re-fetch failed before phase 1, skipping"); - continue; - } - }; - let fresh_repo = match db.get_repo_by_id(&repo.id).await { - Ok(Some(r)) => r, - Ok(None) => { - tracing::warn!(repo = %repo_slug, "repo disappeared from DB before phase 1, skipping"); - continue; - } - Err(e) => { - tracing::warn!(repo = %repo_slug, err = %e, "repo re-fetch failed before phase 1, skipping"); - continue; - } + // Re-check quarantine AND visibility right now (fresh rules + repo row), + // then re-derive the allowed set from those fresh rules so a path-scoped + // narrowing made mid-scan is honored before anything is pinned. + let (fresh_repo, fresh_rules) = match recheck_public_pin(db, &repo.id, &repo_slug).await { + Some(v) => v, + None => continue, }; - if !crate::visibility::listable_at_root( - &rules, - fresh_repo.is_public, - &fresh_repo.owner_did, - None, - ) { - tracing::warn!(repo = %repo_slug, "visibility narrowed mid-scan, skipping phase 1"); - continue; - } // Visibility may have narrowed mid-scan with a path-scoped deny. - // Recompute the allowed set from fresh rules in a spawn_blocking - // and intersect it with the existing object_list (R1-P1, R1-P2). - let fresh_disk = disk.clone(); - let fresh_rules = rules.clone(); - let fresh_owner = fresh_repo.owner_did.clone(); - let fresh_is_public = fresh_repo.is_public; - let existing_list = object_list; - let refilter_ctx = ctx.clone(); - - let refiltered = tokio::time::timeout( - REPO_SCAN_DEADLINE, - tokio::task::spawn_blocking(move || -> anyhow::Result> { - let _guard = crate::git::set_scan_context(refilter_ctx); - let allowed = crate::git::visibility_pack::replicable_blob_set( - &fresh_disk, - &fresh_rules, - fresh_is_public, - &fresh_owner, - )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&fresh_disk)?; - Ok(crate::git::visibility_pack::replicable_objects_fail_closed( - existing_list, - &allowed, - &all_blobs, - )) - }), + // Recompute the allowed set from fresh rules and intersect it with the + // existing object_list. + let refiltered = refilter_public_objects( + &ctx, + &disk, + &fresh_rules, + fresh_repo.is_public, + &fresh_repo.owner_did, + object_list, ) .await; - - let object_list: Vec = match refiltered { - Ok(Ok(Ok(list))) => list, - Ok(Ok(Err(e))) => { - tracing::warn!(repo = %repo_slug, err = %e, "fresh-visibility re-filter failed, skipping"); - continue; - } - Ok(Err(e)) => { - tracing::warn!(repo = %repo_slug, err = %e, "fresh-visibility re-filter task panicked, skipping"); - continue; - } - Err(_) => { - ctx.canceled.store(true, Ordering::SeqCst); - #[cfg(unix)] - { - let pgids: Vec = ctx - .registry - .lock() - .unwrap_or_else(|e| e.into_inner()) - .iter() - .copied() - .collect(); - for &pgid in &pgids { - unsafe { - let _ = libc::kill(-pgid, libc::SIGTERM); - } - } - } - tracing::warn!(repo = %repo_slug, "fresh-visibility re-filter deadline exceeded, skipped"); - continue; - } + let Some(object_list) = refiltered else { + tracing::warn!(repo = %repo_slug, "fresh-visibility re-filter failed, skipping"); + continue; }; + if object_list.is_empty() { + continue; + } - let _ipfs_enabled = !config.ipfs_api.is_empty(); + let ipfs_enabled = !config.ipfs_api.is_empty(); let pinata_enabled = !config.pinata_jwt.is_empty(); - // IPFS-missing set (capped). - let already_ipfs = match db.filter_ipfs_pinned_oids(&object_list).await { - Ok(v) => v, - Err(e) => { - tracing::warn!(repo = %repo_slug, err = %e, "filter_ipfs_pinned_oids failed, skipping"); - continue; - } - }; - let ipfs_missing: Vec = { - let all_set: HashSet<&str> = object_list.iter().map(|s| s.as_str()).collect(); - let done_set: HashSet<&str> = already_ipfs.iter().map(|s| s.as_str()).collect(); - let mut v: Vec = all_set - .difference(&done_set) - .map(|s| s.to_string()) - .collect(); - if v.len() > MAX_OBJECTS_PER_REPO { - v.truncate(MAX_OBJECTS_PER_REPO); - tracing::warn!( - repo = %repo_slug, - cap = MAX_OBJECTS_PER_REPO, - "IPFS per-repo missing cap reached, truncating" - ); + // IPFS-missing set. A filter DB error skips only the IPFS gap-fill and + // lets the Pinata path still run (R1-P3), instead of dropping the repo. + let ipfs_missing: Vec = if ipfs_enabled { + match db.filter_ipfs_pinned_oids(&object_list).await { + Ok(already) => { + cap_missing(missing_oids(&object_list, &already), &repo_slug, "IPFS") + } + Err(e) => { + tracing::warn!(repo = %repo_slug, err = %e, "filter_ipfs_pinned_oids failed, IPFS gap-fill skipped this pass"); + Vec::new() + } } - v + } else { + Vec::new() }; let pinata_missing: Vec = if pinata_enabled { - let already = match db.filter_pinata_pinned_oids(&object_list).await { - Ok(v) => v, + match db.filter_pinata_pinned_oids(&object_list).await { + Ok(already) => { + cap_missing(missing_oids(&object_list, &already), &repo_slug, "Pinata") + } Err(e) => { - tracing::warn!(repo = %repo_slug, err = %e, "filter_pinata_pinned_oids failed, skipping"); - continue; + tracing::warn!(repo = %repo_slug, err = %e, "filter_pinata_pinned_oids failed, Pinata gap-fill skipped this pass"); + Vec::new() } - }; - let all_set: HashSet<&str> = object_list.iter().map(|s| s.as_str()).collect(); - let done_set: HashSet<&str> = already.iter().map(|s| s.as_str()).collect(); - let mut v: Vec = all_set - .difference(&done_set) - .map(|s| s.to_string()) - .collect(); - if v.len() > MAX_OBJECTS_PER_REPO { - v.truncate(MAX_OBJECTS_PER_REPO); - tracing::warn!( - repo = %repo_slug, - cap = MAX_OBJECTS_PER_REPO, - "Pinata per-repo missing cap reached, truncating" - ); } - v } else { Vec::new() }; - let gaps_ipfs = ipfs_missing.len(); - let gaps_pinata = pinata_missing.len(); - let repo_gaps = gaps_ipfs + gaps_pinata; + // Count UNIQUE missing objects across both backends (R1-P3): an object + // absent from both must not be counted twice. + let mut gap_union: HashSet<&str> = HashSet::new(); + gap_union.extend(ipfs_missing.iter().map(|s| s.as_str())); + gap_union.extend(pinata_missing.iter().map(|s| s.as_str())); + let repo_gaps = gap_union.len(); if repo_gaps > 0 { total_gaps_found += repo_gaps; crate::metrics::record_reconciliation_gaps_found(repo_gaps as u64); } - let pinned_ipfs = match tokio::time::timeout( - PIN_PHASE_DEADLINE, - crate::ipfs_pin::pin_new_objects(&config.ipfs_api, &disk, ipfs_missing, db), - ) - .await - { - Ok(v) => v, - Err(_) => { - tracing::warn!(repo = %repo_slug, "IPFS pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + // Re-validate quarantine + visibility IMMEDIATELY before each backend + // pin (R1-P1): for content-addressed public pins a stale allow is + // effectively irreversible, and the pin itself takes time. + let pinned_ipfs: Vec<(String, String)> = if ipfs_enabled && !ipfs_missing.is_empty() { + if recheck_public_pin(db, &repo.id, &repo_slug).await.is_none() { Vec::new() + } else { + match tokio::time::timeout( + PIN_PHASE_DEADLINE, + crate::ipfs_pin::pin_new_objects(&config.ipfs_api, &disk, ipfs_missing, db), + ) + .await + { + Ok(v) => v, + Err(_) => { + tracing::warn!(repo = %repo_slug, "IPFS pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + Vec::new() + } + } } + } else { + Vec::new() }; - let pinned_pinata = match tokio::time::timeout( - PIN_PHASE_DEADLINE, - crate::pinata::pin_new_objects( - http_client, - &config.pinata_upload_url, - &config.pinata_jwt, - &disk, - pinata_missing, - db, - ), - ) - .await - { - Ok(v) => v, - Err(_) => { - tracing::warn!(repo = %repo_slug, "Pinata pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + let pinned_pinata: Vec<(String, String)> = if pinata_enabled && !pinata_missing.is_empty() { + if recheck_public_pin(db, &repo.id, &repo_slug).await.is_none() { Vec::new() + } else { + match tokio::time::timeout( + PIN_PHASE_DEADLINE, + crate::pinata::pin_new_objects( + http_client, + &config.pinata_upload_url, + &config.pinata_jwt, + &disk, + pinata_missing, + db, + ), + ) + .await + { + Ok(v) => v, + Err(_) => { + tracing::warn!(repo = %repo_slug, "Pinata pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + Vec::new() + } + } } + } else { + Vec::new() }; + // `pin_new_objects` returns only objects whose DB record was written + // (R1-P3), so a backend that uploaded bytes but failed to persist is + // not counted as "filled". let repo_filled = pinned_ipfs.len() + pinned_pinata.len(); if repo_filled > 0 { total_gaps_filled += repo_filled; @@ -464,54 +607,21 @@ async fn run_pass( // ── Phase 2: Encrypted recovery-copy resealing (withheld blobs) ── - // Recheck quarantine AND visibility before encrypted pinning (P2). - match db.is_repo_quarantined(&repo.id).await { - Ok(true) => { - tracing::warn!(repo = %repo_slug, "repo quarantined, skipping encrypted pinning"); - continue; - } - Ok(false) => {} - Err(e) => { - tracing::warn!(repo = %repo_slug, err = %e, "quarantine recheck failed, skipping encrypted pin"); - continue; - } - } - let rules = match db.list_visibility_rules(&repo.id).await { - Ok(r) => r, - Err(e) => { - tracing::warn!(repo = %repo_slug, err = %e, "visibility rules re-fetch failed before phase 2, skipping"); - continue; - } + // Recheck quarantine AND root visibility before encrypted pinning, using + // FRESH repo identity (R1-P2): the batch snapshot may predate a narrow. + let (fresh_repo2, fresh_rules2) = match recheck_public_pin(db, &repo.id, &repo_slug).await { + Some(v) => v, + None => continue, }; - let fresh_repo = match db.get_repo_by_id(&repo.id).await { - Ok(Some(r)) => r, - Ok(None) => { - tracing::warn!(repo = %repo_slug, "repo disappeared from DB before phase 2, skipping"); - continue; - } - Err(e) => { - tracing::warn!(repo = %repo_slug, err = %e, "repo re-fetch failed before phase 2, skipping"); - continue; - } - }; - if !crate::visibility::listable_at_root( - &rules, - fresh_repo.is_public, - &fresh_repo.owner_did, - None, - ) { - tracing::warn!(repo = %repo_slug, "visibility narrowed mid-scan, skipping phase 2"); - continue; - } - let has_path_scoped = crate::git::visibility_pack::has_path_scoped_rule(&rules); - if has_path_scoped && !config.ipfs_api.is_empty() { + let has_path_scoped = crate::git::visibility_pack::has_path_scoped_rule(&fresh_rules2); + if has_path_scoped && ipfs_enabled { let ctx2 = crate::git::ScanContext::new(); let ctx2_clone = ctx2.clone(); let p = disk.clone(); - let owner = repo.owner_did.clone(); - let r = rules.clone(); - let is_public_2 = repo.is_public; + let owner = fresh_repo2.owner_did.clone(); + let r = fresh_rules2.clone(); + let is_public_2 = fresh_repo2.is_public; let recipients = tokio::time::timeout( REPO_SCAN_DEADLINE, tokio::task::spawn_blocking(move || { @@ -543,22 +653,7 @@ async fn run_pass( continue; } Err(_) => { - ctx2.canceled.store(true, Ordering::SeqCst); - #[cfg(unix)] - { - let pgids: Vec = ctx2 - .registry - .lock() - .unwrap_or_else(|e| e.into_inner()) - .iter() - .copied() - .collect(); - for &pgid in &pgids { - unsafe { - let _ = libc::kill(-pgid, libc::SIGTERM); - } - } - } + escalate_kill(&ctx2, "encrypted recovery deadline exceeded"); tracing::warn!( repo = %repo_slug, "encrypted recovery deadline exceeded, killed active git subprocesses, skipping" @@ -568,16 +663,33 @@ async fn run_pass( }; if !rec.is_empty() { - let sealed = crate::encrypted_pin::encrypt_and_pin( - &config.ipfs_api, - &disk, - db, - &repo.id, - node_seed, - &rec, + // Bound the seal+pin work (R1-P2): an unavailable backend must + // not hold the sweep past the pin-phase budget. + let sealed = tokio::time::timeout( + PIN_PHASE_DEADLINE, + crate::encrypted_pin::encrypt_and_pin( + &config.ipfs_api, + &disk, + db, + &repo.id, + node_seed, + &rec, + ), ) .await; + let sealed: Vec<(String, String)> = match sealed { + Ok(v) => v, + Err(_) => { + tracing::warn!( + repo = %repo_slug, + "encrypted pin phase timed out after {:?}", + PIN_PHASE_DEADLINE + ); + Vec::new() + } + }; + // Anchor only when something was newly sealed this pass. // This avoids unbounded Irys writes on a timer — repos // with no withheld changes do not re-anchor the manifest. @@ -612,7 +724,16 @@ async fn run_pass( } } - Ok((batch.len(), total_gaps_found, total_gaps_filled)) + // Persist the cursor only when the WHOLE batch completed. If shutdown + // interrupted us, leave the persisted cursor at the previous batch's end so + // the next run re-walks the unprocessed tail (R2-P1, R1-P3). + if batch_completed { + if let Err(e) = db.set_node_state(CURSOR_KEY, Some(&batch_last)).await { + tracing::warn!(err = %e, "failed to persist reconciliation sweep cursor"); + } + } + + Ok((repos_scanned, total_gaps_found, total_gaps_filled)) } #[cfg(test)] @@ -672,9 +793,9 @@ mod tests { assert!(!super::should_spawn(&cfg)); } - /// spawn() must return immediately (without panicking or touching the DB) - /// when neither IPFS nor Pinata is configured. This proves the gate - /// branch at the top of spawn() is actually reachable. + /// spawn() must return `false` (and not spawn a task, touch the DB, or + /// panic) when neither IPFS nor Pinata is configured. This proves the gate + /// branch at the top of spawn() is actually reachable and observable. #[tokio::test] async fn test_spawn_gate_skips_when_no_pin_backends_configured() { let config = empty_pin_config(); @@ -693,9 +814,55 @@ mod tests { let node_did = kp.did(); let (_tx, rx) = watch::channel(false); - // spawn() should return synchronously (no tokio::spawn) and never + // spawn() should return false synchronously (no tokio::spawn) and never // await the DB. The test completes without timeout == gate is live. - super::spawn(db, config, http, kp, node_did, rx); + assert!( + !super::spawn(db, config, http, kp, node_did, rx), + "gated spawn must report it did not start a worker" + ); + } + + /// spawn() returns true and starts a worker when a backend is configured; + /// the caller uses that to gate its own "worker started" logging. + #[tokio::test] + async fn test_spawn_returns_true_when_ipfs_configured() { + let config = ipfs_config(); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect_lazy("postgresql://localhost/gitlawb_test_nonexistent") + .unwrap(); + let db = std::sync::Arc::new(crate::db::Db::for_testing(pool)); + let http = std::sync::Arc::new(reqwest::Client::new()); + let kp = std::sync::Arc::new(gitlawb_core::identity::Keypair::generate()); + let node_did = kp.did(); + let (_tx, rx) = watch::channel(false); + + assert!( + super::spawn(db, config, http, kp, node_did, rx), + "configured spawn must report it started a worker" + ); + } + + /// The missing set must be deterministic, which is what makes the sweep's + /// per-repo pin order reproducible across passes. The cap is applied by + /// `cap_missing` at the call site, so `missing_oids` stays uncapped. + #[test] + fn missing_oids_is_deterministic() { + let all = vec![ + "c".to_string(), + "a".to_string(), + "b".to_string(), + "d".to_string(), + ]; + let done = vec!["b".to_string()]; + + let first = super::missing_oids(&all, &done); + let second = super::missing_oids(&all, &done); + assert_eq!(first, second, "missing set must be deterministic"); + assert_eq!( + first, + vec!["a".to_string(), "c".to_string(), "d".to_string()] + ); } /// Constant smoke-check kept as a compile-time tripwire. @@ -703,4 +870,213 @@ mod tests { fn sweep_interval_constant_is_nonzero() { assert_ne!(super::SWEEP_INTERVAL_SECS, 0); } + + // ── run_pass integration tests ──────────────────────────────────────── + + /// Minimal git repo builder (mirrors push_delta's test helper). + struct Repo { + _td: tempfile::TempDir, + path: std::path::PathBuf, + } + + impl Repo { + fn new() -> Self { + let td = tempfile::TempDir::new().unwrap(); + let path = td.path().to_path_buf(); + let r = Repo { _td: td, path }; + r.git(&["init", "-q", "-b", "main"]); + r.git(&["config", "user.email", "t@t"]); + r.git(&["config", "user.name", "t"]); + r + } + + fn git(&self, args: &[&str]) -> String { + let out = std::process::Command::new("git") + .args(args) + .current_dir(&self.path) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + fn commit_file(&self, name: &str, body: &str) -> String { + std::fs::write(self.path.join(name), body).unwrap(); + self.git(&["add", name]); + self.git(&["commit", "-qm", &format!("add {name}")]); + self.git(&["rev-parse", "HEAD"]) + } + } + + fn seed_repo(owner: &str, name: &str, disk_path: &str) -> crate::db::RepoRecord { + let now = chrono::Utc::now(); + crate::db::RepoRecord { + id: uuid::Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: disk_path.to_string(), + forked_from: None, + machine_id: None, + } + } + + /// The sweep must repair an IPFS durability gap end to end: a public repo + /// whose objects were never pinned gets every reachable blob pinned and + /// recorded (R2-P2 "test the behavior the PR exists to change"). + #[sqlx::test] + async fn sweep_fills_ipfs_gap_and_persists_cursor(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("a.txt", "public blob\n"); + + let rec = seed_repo( + "did:key:zSweepOwner", + "sweep-repo", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // Mock IPFS: every /api/v0/add returns a fixed CID. mockito's unified + // matcher compares the full "path?query" target, so the query string + // pin_git_object appends must be part of the mock path. + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .expect_at_least(1) + .with_status(200) + .with_body(r#"{"Hash":"QmSweepMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + assert_eq!(scanned, 1, "one repo scanned"); + assert!(gaps >= 1, "at least one missing blob found"); + assert_eq!( + filled, gaps, + "every found gap is filled in a clean mock-backed run" + ); + _m.assert_async().await; + + // The recorded pin makes the blob "already done" on the next pass. + let blob = repo_on_disk.git(&["rev-parse", "HEAD:a.txt"]); + assert!( + db.has_ipfs_cid(&blob).await.unwrap(), + "pinned CID must be recorded and classified as IPFS-pinned" + ); + + // Cursor persisted in node_state so a restart resumes, not re-walks. + let persisted = db + .get_node_state(super::CURSOR_KEY) + .await + .unwrap() + .expect("cursor must be persisted after a completed batch"); + assert_eq!(persisted, rec.id, "cursor equals the last batch repo id"); + + // Second pass: no gaps remain. + let (_, gaps2, filled2) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + assert_eq!(gaps2, 0, "second pass finds no remaining gaps"); + assert_eq!(filled2, 0); + } + + /// Mirror rows (slash-form id, hardcoded is_public=true, no replicated + /// visibility rules) must be skipped entirely: sweeping one would + /// irreversibly publish content the canonical gate never admitted (R2-P1). + #[sqlx::test] + async fn sweep_skips_mirror_rows(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("secret.txt", "must not be published\n"); + + // A mirror row pointing at a real, public-on-disk repo. + db.upsert_mirror_repo( + "zMirrorOwner", + "mirror-repo", + &repo_on_disk.path.display().to_string(), + None, + false, + ) + .await + .unwrap(); + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + "http://127.0.0.1:1", // unreachable; must never be hit + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + assert_eq!(scanned, 0, "mirror row is not scanned"); + assert_eq!(gaps, 0, "mirror row produces no gaps"); + assert_eq!(filled, 0, "mirror row is never pinned"); + + // Nothing was recorded for the mirror's content. + assert!( + db.list_pinned_cids().await.unwrap().is_empty(), + "no pinned_cids rows may exist after a mirror-only pass" + ); + } } From c1a91c01bb7e17b45b0e5f2333fae4ccf6216813 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Fri, 7 Aug 2026 17:22:59 +0600 Subject: [PATCH 12/26] fix(git): release registry lock before waiting and route cat-file via GitCommand - Hold the process-group registry lock only for the check+register decision, never across wait_with_output, so a zombie-reap that outlives the deadline cannot block other threads (R1-P2) - Route object_type and read_object_content through GitCommand so cat-file invoked from inside a sweep's blocking scan is registered and killed on deadline (R1-P3) --- crates/gitlawb-node/src/git/mod.rs | 41 +++++++++++++++++++--------- crates/gitlawb-node/src/git/store.rs | 10 ++++--- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/crates/gitlawb-node/src/git/mod.rs b/crates/gitlawb-node/src/git/mod.rs index dd6a7283..013e75fb 100644 --- a/crates/gitlawb-node/src/git/mod.rs +++ b/crates/gitlawb-node/src/git/mod.rs @@ -190,26 +190,41 @@ impl GitCommand { // interleaving between the check and the insert — if canceled // is set while we hold the lock, the sweep cannot drain the // registry until we release it. - if let Some(ref ctx) = ctx { + // + // The lock is held ONLY for the check + register decision, never + // across `wait_with_output` below: a zombie-reap that outlives the + // deadline must not block other threads from registering or draining + // their pgids (R1-P2). + let kill_after_cancel = if let Some(ref ctx) = ctx { let mut registry = ctx.registry.lock().unwrap_or_else(|e| e.into_inner()); if ctx.canceled.load(Ordering::SeqCst) { - // Canceled after spawn: kill the whole process group (not - // just the immediate child) and wait to avoid zombies. + true + } else { if let Some(pgid) = pgid { - #[cfg(unix)] - unsafe { - let _ = libc::kill(-pgid, libc::SIGTERM); - } + registry.insert(pgid); } - let _ = child.wait_with_output(); - return Err(io::Error::new( - io::ErrorKind::TimedOut, - "scan canceled after spawn", - )); + false } + } else { + false + }; + + if kill_after_cancel { + // Canceled after spawn: kill the whole process group (not just the + // immediate child) and wait to avoid zombies. The pgid was never + // registered (we bailed before the insert), so no registry cleanup + // is owed here. if let Some(pgid) = pgid { - registry.insert(pgid); + #[cfg(unix)] + unsafe { + let _ = libc::kill(-pgid, libc::SIGTERM); + } } + let _ = child.wait_with_output(); + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "scan canceled after spawn", + )); } let guard = PgidGuard { pgid, ctx }; diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index ceb75531..b0efffb6 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -271,10 +271,13 @@ pub struct TreeEntry { /// `gitlawb_core::cid::Cid::from_git_object_bytes`. /// /// Get just the object type. Returns `None` if the object doesn't exist. +/// +/// Runs through [`crate::git::GitCommand`] so a cat-file invoked from inside a +/// reconciliation sweep's blocking scan is registered in the scan context and +/// killed on deadline, exactly like the pack/cat-file commands it calls (R1-P3). pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> { - let type_output = Command::new("git") + let type_output = crate::git::GitCommand::new(repo_path) .args(["cat-file", "-t", sha256_hex]) - .current_dir(repo_path) .output() .context("failed to run git cat-file -t")?; @@ -291,9 +294,8 @@ pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> /// Read an object's content if its type is already known. pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) -> Result> { - let content_output = Command::new("git") + let content_output = crate::git::GitCommand::new(repo_path) .args(["cat-file", obj_type, sha256_hex]) - .current_dir(repo_path) .output() .context("failed to run git cat-file ")?; From 7d570bea8d633271b962240a55d708d6991f62eb Mon Sep 17 00:00:00 2001 From: Gravirei Date: Fri, 7 Aug 2026 17:23:02 +0600 Subject: [PATCH 13/26] fix(pin): count only DB-persisted pins as filled An object whose upload succeeded but whose DB record failed is not durably pinned; counting it would overstate the sweep's repair (R1-P3). Push to the returned vec only after record_pinned_cid / record_pinata_cid succeeds. --- crates/gitlawb-node/src/ipfs_pin.rs | 11 +++++++---- crates/gitlawb-node/src/pinata.rs | 11 ++++++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index f48748be..6118361d 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -94,6 +94,9 @@ pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { /// lockstep. /// /// Returns a list of `(sha256_hex, cid)` pairs for objects pinned this call. +/// An object whose upload succeeded but whose DB record failed is NOT included: +/// it is not durably pinned, so counting it as "filled" would overstate the +/// sweep's repair (R1-P3). pub async fn pin_new_objects( ipfs_api: &str, repo_path: &std::path::Path, @@ -130,12 +133,12 @@ pub async fn pin_new_objects( // Pin to IPFS match pin_git_object(ipfs_api, &sha, &data).await { - Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinned_cid(&sha, &cid).await { + Ok(cid) if !cid.is_empty() => match db.record_pinned_cid(&sha, &cid).await { + Ok(()) => pinned.push((sha, cid)), + Err(e) => { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); } - pinned.push((sha, cid)); - } + }, Ok(_) => {} Err(e) => { tracing::warn!(sha = %sha, err = %e, "failed to pin git object to IPFS"); diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 6c9c0bff..82efd92a 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -111,10 +111,15 @@ pub async fn pin_new_objects( match pin_object(client, upload_url, jwt, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinata_cid(&sha, &cid).await { - tracing::warn!(sha = %sha, err = %e, "failed to record pinata_cid in DB"); + // Only a successfully-persisted record counts as pinned; an + // upload that failed to reach the DB is not durably pinned + // (R1-P3). + match db.record_pinata_cid(&sha, &cid).await { + Ok(()) => pinned.push((sha, cid)), + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "failed to record pinata_cid in DB"); + } } - pinned.push((sha, cid)); } Ok(_) => {} Err(e) => { From 984b38ac177fb84382c2493422d7f8267767dae8 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Fri, 7 Aug 2026 17:23:06 +0600 Subject: [PATCH 14/26] fix(api): stop leaking pinata_cid and skip rows with no CID at all Rows with neither a local nor a Pinata CID are omitted so the cid field stays an always-string field, and the raw pinata_cid is no longer surfaced: it is node-internal state and leaking it to unauthenticated callers exposes infrastructure detail (R1-P3). --- crates/gitlawb-node/src/api/ipfs.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 41aa6ce5..942e17bc 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -216,8 +216,13 @@ pub async fn get_by_cid( /// Returns all CIDs that have been pinned from git objects received via push. /// Each entry includes the git SHA-256 hex, a CIDv1 string, and the timestamp /// when it was pinned. For Pinata-only rows (no local IPFS pin), the `cid` -/// field carries `pinata_cid` so CLI consumers see a usable value. The raw -/// `pinata_cid` is also surfaced. +/// field carries `pinata_cid` so CLI consumers see a usable value. +/// +/// The raw `pinata_cid` is deliberately NOT surfaced: it is node-internal state +/// (which Pinata identity a blob was uploaded with) and exposing it to +/// unauthenticated callers leaks infrastructure detail (R1-P3). Rows with +/// neither a local nor a Pinata CID are omitted so `cid` stays an +/// always-string field. pub async fn list_pins(State(state): State) -> Result> { let pins = state .db @@ -227,6 +232,7 @@ pub async fn list_pins(State(state): State) -> Result = pins .into_iter() + .filter(|p| p.cid.is_some() || p.pinata_cid.is_some()) .map(|p| { // Synthesize a usable CID from pinata_cid when this is a // Pinata-only row (no local IPFS pin). @@ -235,7 +241,6 @@ pub async fn list_pins(State(state): State) -> Result Date: Fri, 7 Aug 2026 17:23:09 +0600 Subject: [PATCH 15/26] fix(node): gate sweep-started log on spawn and qualify backstop wording - Only log 'reconciliation sweep worker started' when spawn actually started a worker (returns true) - Assert the reconciliation gaps_found/gaps_filled counters in the metrics encode test - Qualify the push_delta backstop comment: a node with the sweep disabled or no pin backend has no durability backstop --- crates/gitlawb-node/src/git/push_delta.rs | 10 ++++++---- crates/gitlawb-node/src/main.rs | 5 +++-- crates/gitlawb-node/src/metrics.rs | 10 ++++++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 9433ab8b..11257246 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -262,10 +262,12 @@ pub struct PinCandidateSet { /// Every degraded path is **logged**, not silent: a full-scan fallback, a /// failed full scan, and a panicked blocking task each emit a warning. On a /// failed full scan or a task panic the candidate set is empty (pin nothing -/// this push); that is a durability gap the reconciliation sweep backstops, and -/// it can never leak because the withheld/fail-closed filter still runs on -/// whatever set is returned. `full_scan` rides on the returned set so the caller -/// knows when the dangling-inclusive filter is required. +/// this push); that is a durability gap the reconciliation sweep backstops +/// when it is enabled and a pin backend is configured (a node running with the +/// sweep disabled or with no IPFS/Pinata backend has no backstop), and it can +/// never leak because the withheld/fail-closed filter still runs on whatever +/// set is returned. `full_scan` rides on the returned set so the caller knows +/// when the dangling-inclusive filter is required. pub async fn resolve_candidates_for_push( repo_path: PathBuf, new_tips: Vec, diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index dc372f05..511bf23d 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -508,8 +508,9 @@ async fn main() -> Result<()> { let node_keypair = Arc::clone(&state.node_keypair); let node_did = state.node_did.clone(); let shutdown_rx = state.subscribe_shutdown(); - reconciliation::spawn(db, config, http_client, node_keypair, node_did, shutdown_rx); - info!("reconciliation sweep worker started"); + if reconciliation::spawn(db, config, http_client, node_keypair, node_did, shutdown_rx) { + info!("reconciliation sweep worker started"); + } } // On-chain operator setup: verify stake + spawn heartbeat loop diff --git a/crates/gitlawb-node/src/metrics.rs b/crates/gitlawb-node/src/metrics.rs index bcb13044..d1a85c83 100644 --- a/crates/gitlawb-node/src/metrics.rs +++ b/crates/gitlawb-node/src/metrics.rs @@ -348,6 +348,8 @@ mod tests { .expect("PUSHES set after init") .with_label_values(&["alice/repo"]) .inc(); + record_reconciliation_gaps_found(7); + record_reconciliation_gaps_filled(3); let body = encode().expect("encode should succeed after init"); assert!( @@ -362,6 +364,14 @@ mod tests { body.contains("gitlawb_pushes_total{repo=\"alice/repo\"} 1"), "expected the incremented counter to be visible in: {body}" ); + assert!( + body.contains("gitlawb_reconciliation_gaps_found_total 7"), + "expected the reconciliation gaps-found counter to be visible in: {body}" + ); + assert!( + body.contains("gitlawb_reconciliation_gaps_filled_total 3"), + "expected the reconciliation gaps-filled counter to be visible in: {body}" + ); } #[test] From e5beeb9ddffd3686be2eef924a2b6ad19d7bfe55 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Fri, 7 Aug 2026 17:23:13 +0600 Subject: [PATCH 16/26] docs: document GITLAWB_RECONCILIATION_SWEEP --- .env.example | 7 +++++++ README.md | 1 + 2 files changed, 8 insertions(+) diff --git a/.env.example b/.env.example index bbd9a342..a0d2b770 100644 --- a/.env.example +++ b/.env.example @@ -151,6 +151,13 @@ GITLAWB_TRUSTED_PROXY= # Enable automatic background sync from known peers GITLAWB_AUTO_SYNC=false +# ── Reconciliation sweep ───────────────────────────────────────────────── +# Periodic durability sweep: re-derives the public pin set and the withheld-blob +# recovery set each hour and fills gaps so a dropped replication job never means +# data loss. Defaults to true; set to false to disable the sweep even when a pin +# backend (IPFS/Pinata) is configured. +GITLAWB_RECONCILIATION_SWEEP=true + # ── iCaptcha proof-of-intelligence gate ─────────────────────────────────── # Optional gate on create_repo + register: require callers to present an # iCaptcha proof (X-ICaptcha-Proof header) earned at icaptcha.gitlawb.com. diff --git a/README.md b/README.md index 57ce2885..0ce43dde 100644 --- a/README.md +++ b/README.md @@ -342,6 +342,7 @@ Important node settings: | `GITLAWB_BOOTSTRAP_DISABLE_SEEDS` | Disable embedded seed peers for isolated dev/test networks. | | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | +| `GITLAWB_RECONCILIATION_SWEEP` | Enable the hourly durability sweep that re-pins/backstops missing objects (default `true`; disabled when no IPFS/Pinata backend is configured). | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | | `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack/receive-pack may run before it is aborted (504). Default 600. Does not bound `info/refs` or the withheld-blob path. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | From c868820d50688659903ef03b72d3cfde83c4c000 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Mon, 10 Aug 2026 20:12:32 +0600 Subject: [PATCH 17/26] fix(node): harden reconciliation sweep against visibility narrows Re-authorize the to-pin list at the pin boundary from rules re-fetched at that moment instead of pinning the scan-time set: a path-scoped deny that lands after the mid-scan refilter now suppresses the blob instead of being published in cleartext on IPFS/Pinata. Bind the encrypted manifest anchor to the fresh repo identity, bound public objects to ref-reachability so dangling objects from an aborted push are never pinned, share one scan deadline across the walk/refilter/boundary stages, and clear the cursor on a short final page. Adds a must-not-pin test for a withheld subtree. --- crates/gitlawb-node/src/git/push_delta.rs | 31 +++ crates/gitlawb-node/src/reconciliation.rs | 318 +++++++++++++++++----- 2 files changed, 282 insertions(+), 67 deletions(-) diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 5a592043..704cf74b 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -209,6 +209,37 @@ pub fn list_all_objects(repo_path: &Path, git_bin: &str, deadline: Instant) -> R .collect()) } +/// The set of objects reachable from any ref, via +/// `git rev-list --all --objects --no-object-names`. +/// +/// The full-object-database enumeration ([`list_all_objects`]) contains +/// dangling commits, trees, and blobs (`git cat-file --batch-all-objects` lists +/// loose objects from an aborted or still-running push). Blob candidates are +/// already fail-closed against the reachable, visibility-allowed set — but +/// commits and trees have no path scoping to fail closed against, so the sweep +/// must bound them to ref-reachability or an unreferenced commit's message, +/// author, and parent links (and any unreferenced tree) would be published to a +/// public IPFS/Pinata endpoint. This is that reachability bound. +pub fn reachable_object_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &["rev-list", "--all", "--objects", "--no-object-names"], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); + Ok(stdout + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect()) +} + /// Like [`list_all_objects`] but pairs each OID with its object type, via /// `--batch-check='%(objectname) %(objecttype)'`. The pin path's fail-closed /// filter needs to tell blobs (content, withholdable) from commits/trees diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs index 7edc1b8a..b1682590 100644 --- a/crates/gitlawb-node/src/reconciliation.rs +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -126,26 +126,30 @@ pub fn spawn( /// runs against rules re-fetched after the git scan, so a narrowing made /// mid-scan is honored before anything is pinned. /// -/// The two `spawn_blocking` stages (full scan, re-filter) share the deadline so -/// the total blocking time per repo stays bounded. +/// The full scan and this re-filter carry the same absolute `deadline` so the +/// total blocking time per repo stays bounded: `run_pass` computes one deadline +/// and hands it to both stages, and each stage runs against the remaining +/// budget rather than a fresh full timeout (the two stages share the document +/// per-repo cap, not two independent ones). async fn refilter_public_objects( disk: &std::path::Path, rules: &[crate::db::VisibilityRule], is_public: bool, owner_did: &str, object_list: Vec, + deadline: Instant, ) -> Option> { let disk_clone = disk.to_path_buf(); let rules_clone = rules.to_vec(); let owner_clone = owner_did.to_string(); match tokio::time::timeout( - REPO_SCAN_DEADLINE, + deadline.saturating_duration_since(Instant::now()), tokio::task::spawn_blocking(move || -> anyhow::Result> { - // One deadline spans the whole re-filter (replicable_blob_set_bounded - // + all_blob_oids), so a slow walk is bounded as a unit rather than - // granting each git child a fresh timeout. - let deadline = Instant::now() + REPO_SCAN_DEADLINE; + // The shared deadline spans this whole re-filter + // (replicable_blob_set_bounded + all_blob_oids), so a slow walk is + // bounded as a unit rather than granting each git child a fresh + // timeout. let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( &disk_clone, "git", @@ -358,34 +362,49 @@ async fn run_pass( } // ── Full git scan (bounded) ───────────────────────────────────── + // One absolute deadline spans the whole scan AND the mandatory + // visibility re-filter below, so a repo's total blocking time stays + // bounded at REPO_SCAN_DEADLINE (they share the cap, not two + // independent ones). Each stage runs against the remaining budget. + let scan_deadline = Instant::now() + REPO_SCAN_DEADLINE; let disk_clone = disk.clone(); let owner_clone = repo.owner_did.clone(); let rules_clone = rules.clone(); let is_public = repo.is_public; let object_list = tokio::time::timeout( - REPO_SCAN_DEADLINE, + scan_deadline.saturating_duration_since(Instant::now()), tokio::task::spawn_blocking(move || -> anyhow::Result> { - // One deadline spans the whole scan (list_all_objects, - // replicable_blob_set_bounded, all_blob_oids), so a slow walk is - // bounded as a unit rather than granting each git child a fresh - // timeout. - let deadline = Instant::now() + REPO_SCAN_DEADLINE; let all_objs = - crate::git::push_delta::list_all_objects(&disk_clone, "git", deadline)?; + crate::git::push_delta::list_all_objects(&disk_clone, "git", scan_deadline)?; let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( &disk_clone, "git", - deadline.saturating_duration_since(Instant::now()), + scan_deadline.saturating_duration_since(Instant::now()), &rules_clone, is_public, &owner_clone, )?; let all_blobs = - crate::git::push_delta::all_blob_oids(&disk_clone, "git", deadline)?; + crate::git::push_delta::all_blob_oids(&disk_clone, "git", scan_deadline)?; + // Fail closed for blobs (already via replicable_objects_fail_closed) + // AND bound commits/trees to ref-reachability: the batch-all-objects + // enumeration carries dangling commits/trees from an aborted push, + // which have no path scoping to fail closed against. Requiring + // membership in the reachable object set keeps their messages, + // authors, parent links, and tree/file-name metadata off public pin + // backends (R2). + let reachable = crate::git::push_delta::reachable_object_oids( + &disk_clone, + "git", + scan_deadline, + )?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( all_objs, &allowed, &all_blobs, - )) + ) + .into_iter() + .filter(|oid| reachable.contains(oid)) + .collect()) }), ) .await; @@ -421,13 +440,15 @@ async fn run_pass( // Visibility may have narrowed mid-scan with a path-scoped deny. // Recompute the allowed set from fresh rules and intersect it with the - // existing object_list. + // existing object_list. Shares `scan_deadline` with the full scan so + // the total blocking time per repo stays bounded at REPO_SCAN_DEADLINE. let refiltered = refilter_public_objects( &disk, &fresh_rules, fresh_repo.is_public, &fresh_repo.owner_did, object_list, + scan_deadline, ) .await; let Some(object_list) = refiltered else { @@ -483,29 +504,55 @@ async fn run_pass( } // Re-validate quarantine + visibility IMMEDIATELY before each backend - // pin (R1-P1): for content-addressed public pins a stale allow is - // effectively irreversible, and the pin itself takes time. + // pin (R1-P1) and re-derive the allowed set from the rules read at that + // moment, intersecting it with the to-pin list (R2-P1): for + // content-addressed public pins a stale allow is effectively + // irreversible, and the pin itself takes time. A path-scoped deny that + // landed after the mid-scan refilter (which only checks root listability) + // is honored here because the candidates are intersected with the set + // allowed under the fresh rules, not just root-gated. let pinned_ipfs: Vec<(String, String)> = if ipfs_enabled && !ipfs_missing.is_empty() { - if recheck_public_pin(db, &repo.id, &repo_slug).await.is_none() { - Vec::new() - } else { - match tokio::time::timeout( - PIN_PHASE_DEADLINE, - crate::ipfs_pin::pin_new_objects( - &config.ipfs_api, + match recheck_public_pin(db, &repo.id, &repo_slug).await { + None => Vec::new(), + Some((fresh_repo, fresh_rules)) => { + let to_pin = match refilter_public_objects( &disk, - "git", + &fresh_rules, + fresh_repo.is_public, + &fresh_repo.owner_did, ipfs_missing, - db, - crate::ipfs_pin::PIN_BATCH_BUDGET, - ), - ) - .await - { - Ok(v) => v, - Err(_) => { - tracing::warn!(repo = %repo_slug, "IPFS pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + scan_deadline, + ) + .await + { + Some(list) => list, + None => { + tracing::warn!(repo = %repo_slug, "IPFS pin-boundary re-derivation failed, skipping"); + Vec::new() + } + }; + if to_pin.is_empty() { Vec::new() + } else { + match tokio::time::timeout( + PIN_PHASE_DEADLINE, + crate::ipfs_pin::pin_new_objects( + &config.ipfs_api, + &disk, + "git", + to_pin, + db, + crate::ipfs_pin::PIN_BATCH_BUDGET, + ), + ) + .await + { + Ok(v) => v, + Err(_) => { + tracing::warn!(repo = %repo_slug, "IPFS pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + Vec::new() + } + } } } } @@ -514,28 +561,49 @@ async fn run_pass( }; let pinned_pinata: Vec<(String, String)> = if pinata_enabled && !pinata_missing.is_empty() { - if recheck_public_pin(db, &repo.id, &repo_slug).await.is_none() { - Vec::new() - } else { - match tokio::time::timeout( - PIN_PHASE_DEADLINE, - crate::pinata::pin_new_objects( - http_client, - &config.pinata_upload_url, - &config.pinata_jwt, + match recheck_public_pin(db, &repo.id, &repo_slug).await { + None => Vec::new(), + Some((fresh_repo, fresh_rules)) => { + let to_pin = match refilter_public_objects( &disk, - "git", + &fresh_rules, + fresh_repo.is_public, + &fresh_repo.owner_did, pinata_missing, - db, - crate::ipfs_pin::PIN_BATCH_BUDGET, - ), - ) - .await - { - Ok(v) => v, - Err(_) => { - tracing::warn!(repo = %repo_slug, "Pinata pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + scan_deadline, + ) + .await + { + Some(list) => list, + None => { + tracing::warn!(repo = %repo_slug, "Pinata pin-boundary re-derivation failed, skipping"); + Vec::new() + } + }; + if to_pin.is_empty() { Vec::new() + } else { + match tokio::time::timeout( + PIN_PHASE_DEADLINE, + crate::pinata::pin_new_objects( + http_client, + &config.pinata_upload_url, + &config.pinata_jwt, + &disk, + "git", + to_pin, + db, + crate::ipfs_pin::PIN_BATCH_BUDGET, + ), + ) + .await + { + Ok(v) => v, + Err(_) => { + tracing::warn!(repo = %repo_slug, "Pinata pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + Vec::new() + } + } } } } @@ -647,14 +715,18 @@ async fn run_pass( // This avoids unbounded Irys writes on a timer — repos // with no withheld changes do not re-anchor the manifest. if !sealed.is_empty() && !config.irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&repo.owner_did); - let slug = format!("{}/{}", owner_short, repo.name); + // Bind the manifest to the FRESH repo identity re-fetched at + // the pin boundary (`fresh_repo2`), not the batch snapshot: + // a renamed/ownership-changed repo must not anchor encrypted + // recovery copies under a stale owner (R1-P2). + let owner_short = crate::db::normalize_owner_key(&fresh_repo2.owner_did); + let slug = format!("{}/{}", owner_short, fresh_repo2.name); let ts = chrono::Utc::now().to_rfc3339(); let node_did_str = node_did.to_string(); let manifest = crate::arweave::EncryptedManifest { repo: &slug, - owner_did: &repo.owner_did, + owner_did: &fresh_repo2.owner_did, node_did: &node_did_str, timestamp: &ts, blobs: &sealed, @@ -681,7 +753,16 @@ async fn run_pass( // interrupted us, leave the persisted cursor at the previous batch's end so // the next run re-walks the unprocessed tail (R2-P1, R1-P3). if batch_completed { - if let Err(e) = db.set_node_state(CURSOR_KEY, Some(&batch_last)).await { + // A short final page means the whole key space is covered: clear the + // cursor now so the next tick starts a fresh cycle instead of burning + // one pass on an empty batch (persisting `batch_last` here would make + // the next tick scan nothing and only then reset). + if batch.len() < REPOS_PER_PASS { + *cursor = None; + if let Err(e) = db.set_node_state(CURSOR_KEY, None).await { + tracing::warn!(err = %e, "failed to clear reconciliation sweep cursor on final page"); + } + } else if let Err(e) = db.set_node_state(CURSOR_KEY, Some(&batch_last)).await { tracing::warn!(err = %e, "failed to persist reconciliation sweep cursor"); } } @@ -952,13 +1033,19 @@ mod tests { "pinned CID must be recorded and classified as IPFS-pinned" ); - // Cursor persisted in node_state so a restart resumes, not re-walks. - let persisted = db - .get_node_state(super::CURSOR_KEY) - .await - .unwrap() - .expect("cursor must be persisted after a completed batch"); - assert_eq!(persisted, rec.id, "cursor equals the last batch repo id"); + // Cursor cleared on a short final page (R2-P1): with one repo the batch + // is the whole key space, so persisting `batch_last` would just force an + // empty tail pass next tick that scans nothing and then clears. Clearing + // now means the next pass starts a fresh cycle immediately. + let persisted = db.get_node_state(super::CURSOR_KEY).await.unwrap(); + assert!( + persisted.is_none(), + "cursor must be cleared after a fully-completed short final page" + ); + assert!( + cursor.is_none(), + "in-memory cursor follows the persisted one" + ); // Second pass: no gaps remain. let (_, gaps2, filled2) = super::run_pass( @@ -1032,4 +1119,101 @@ mod tests { "no pinned_cids rows may exist after a mirror-only pass" ); } + + /// A public repo with a path-scoped deny must NOT have the withheld blob + /// pinned in cleartext on a public backend (R2-P1 "must not pin"): the root + /// stays listable, so the mid-scan refilter AND the pin-boundary re-derivation + /// are the only layers between a narrowed subtree and irreversible public + /// publication. + #[sqlx::test] + async fn sweep_never_pins_withheld_blob_in_cleartext(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("public.txt", "public content\n"); + // git needs the parent directory to exist before `git add` of a nested + // path; create it, then stage via `git add -A` through the helper. + std::fs::create_dir_all(repo_on_disk.path.join("secret")).unwrap(); + repo_on_disk.commit_file("secret/secret.txt", "must not go public\n"); + + // Blob oids, not commit oids: commits are structural and legitimately + // pinned publicly, so the must-not-pin assertion must key on the blob + // whose content is denied at `secret/secret.txt`. + let public_blob = repo_on_disk.git(&["rev-parse", "HEAD:public.txt"]); + let secret_blob = repo_on_disk.git(&["rev-parse", "HEAD:secret/secret.txt"]); + + let rec = seed_repo( + "did:key:zSweepWithheldOwner", + "sweep-withheld", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // Path-scoped deny with no readers: anonymous is allowed the repo root + // (public) but denied every blob under /secret/**, whose content must + // never reach the public pin backends. + db.set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + &rec.owner_did, + ) + .await + .unwrap(); + + // Mock IPFS: every /api/v0/add returns a fixed CID (matches pin_git_object's + // URL, which appends the cid-version/raw-leaves/pin query). + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .with_status(200) + .with_body(r#"{"Hash":"QmWithheldMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + assert_eq!(scanned, 1, "one repo scanned"); + + assert!(gaps >= 1, "public blob is a real gap"); + let _ = filled; // encrypted/sealed copies do not count toward `filled` + + // The public blob is pinned and recorded as IPFS-pinned. + assert!( + db.has_ipfs_cid(&public_blob).await.unwrap(), + "public blob must be pinned in cleartext" + ); + + // The withheld blob must NOT appear with an IPFS CID -- never pinned in + // cleartext. (`has_ipfs_cid` only matches rows with a non-NULL cid, so an + // encrypted copy recorded under `encrypted_blobs` cannot satisfy it.) + assert!( + !db.has_ipfs_cid(&secret_blob).await.unwrap(), + "withheld blob must never be pinned to a public backend in cleartext" + ); + } } From 673c6e3bc98bca02d64df6f48a53708a5eb53d98 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Wed, 12 Aug 2026 12:43:59 +0600 Subject: [PATCH 18/26] fix(core): reject weak-key Ed25519 signatures with strict verification The shared verify primitives (identity::verify and the attestation verifier) accepted the identity-point forgery: public key A = identity, R = identity, S = 0 satisfies [S]B = R + [k]A for any message under ordinary Ed25519 verification. Since identity::verify backs HTTP request authentication, UCANs, and certificates, that is an authentication bypass. Use verify_strict, which rejects small-order R and public keys. --- crates/gitlawb-attest/src/attestation.rs | 29 +++++++++++++++-- crates/gitlawb-core/src/identity.rs | 40 ++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/crates/gitlawb-attest/src/attestation.rs b/crates/gitlawb-attest/src/attestation.rs index 5adc7411..adfa0f63 100644 --- a/crates/gitlawb-attest/src/attestation.rs +++ b/crates/gitlawb-attest/src/attestation.rs @@ -16,7 +16,7 @@ //! by exact match. use base64::{engine::general_purpose::URL_SAFE_NO_PAD as B64U, Engine}; -use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; +use ed25519_dalek::{Signature, Signer, SigningKey, VerifyingKey}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use crate::error::{Error, Result}; @@ -111,7 +111,7 @@ impl Attestation { .try_into() .map_err(|_| Error::Signature("signature must be 64 bytes".to_string()))?; let sig = Signature::from_bytes(&sig_bytes); - vk.verify(&bytes, &sig) + vk.verify_strict(&bytes, &sig) .map_err(|e| Error::Signature(format!("ed25519: {e}")))?; Ok(vk) @@ -483,6 +483,31 @@ mod tests { assert!(matches!(err, Error::Signature(_))); } + /// The identity-point forgery must be rejected: the shared attestation + /// verifier is a cert-bound provenance gate, so accepting the weak-key + /// signature would let anyone mint a forged attestation that verifies. + /// Strict verification rejects small-order public keys and R (the identity + /// point here), which ordinary verification does not. + #[test] + fn verify_rejects_identity_point_forgery() { + let cert_hash = sample_cert_hash(); + let mut att = dummy_attestation(&fresh(), cert_hash); + + // Public key A = identity point (0,1); signature R = identity, S = 0. + // The equation `[S]B = R + [k]A` then holds for any k and any message. + let identity = [1u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut buf = Vec::with_capacity(34); + buf.extend_from_slice(&ED25519_MULTICODEC); + buf.extend_from_slice(&identity); + att.signer = format!("did:key:{}", multibase::encode(multibase::Base::Base58Btc, &buf)); + let mut sig = [0u8; 64]; + sig[..32].copy_from_slice(&identity); + att.sig = B64U.encode(sig); + + let err = att.verify_signature(cert_hash).unwrap_err(); + assert!(matches!(err, Error::Signature(_))); + } + /// A payload that happens to contain a `cert_hash` field of its own does /// not interfere with the outer binding: the attestation envelope's /// `cert_hash` is the only field consulted by `verify_signature`, and the diff --git a/crates/gitlawb-core/src/identity.rs b/crates/gitlawb-core/src/identity.rs index 511e1203..b4e95240 100644 --- a/crates/gitlawb-core/src/identity.rs +++ b/crates/gitlawb-core/src/identity.rs @@ -77,11 +77,18 @@ impl Keypair { } /// Verify an Ed25519 signature. +/// +/// Strict verification: rejects small-order `R` and small-order public keys +/// (the identity point, and any point of low order). Ordinary `verify` accepts +/// a signature forged with the identity point as the public key plus +/// `R = identity, S = 0`, which verifies for *any* message. `identity::verify` +/// is the shared primitive behind HTTP request authentication, UCANs, and +/// certificates, so weak-key acceptance is an authentication bypass, not a +/// malleability nuance. pub fn verify(verifying_key: &VerifyingKey, msg: &[u8], sig_bytes: &[u8; 64]) -> Result<()> { - use ed25519_dalek::Verifier; let sig = Signature::from_bytes(sig_bytes); verifying_key - .verify(msg, &sig) + .verify_strict(msg, &sig) .map_err(|_| Error::SignatureInvalid) } @@ -209,6 +216,35 @@ mod tests { ); } + /// The identity-point forgery: with public key A = identity, R = identity, + /// and S = 0, the equation `[S]B = R + [k]A` holds for every message, + /// because `[k]·identity = identity`. Ordinary (non-strict) Ed25519 + /// verification accepts it, so the shared `verify` primitive must use + /// strict verification, which rejects small-order R and public keys. + #[test] + fn verify_rejects_identity_point_forgery() { + use ed25519_dalek::Verifier; + // The identity point (0,1) compresses to y = 1 with sign bit 0. + let identity = [1u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let vk = VerifyingKey::from_bytes(&identity).expect("identity point is on the curve"); + let mut sig_bytes = [0u8; 64]; + sig_bytes[..32].copy_from_slice(&identity); + let msg = b"arbitrary message the key owner never signed"; + + // Prove the forged signature satisfies the ordinary verification + // equation, so the strict check below is what actually defends the + // boundary (not a signature that was already invalid everywhere). + assert!( + vk.verify(msg, &Signature::from_bytes(&sig_bytes)).is_ok(), + "identity-point forgery must satisfy ordinary verification (this is why strict is needed)" + ); + + assert!( + verify(&vk, msg, &sig_bytes).is_err(), + "strict verification must reject the identity-point forgery" + ); + } + #[test] fn signed_payload_round_trip() { let kp = Keypair::generate(); From 9de685973f6cade7ed5c578fe52be85bf279569b Mon Sep 17 00:00:00 2001 From: Gravirei Date: Wed, 12 Aug 2026 12:44:02 +0600 Subject: [PATCH 19/26] fix(node): stop leaking sqlx/anyhow detail in 500 error bodies (#226) AppError::Db and AppError::Internal serialized the raw error string into the HTTP body, exposing query text and schema details on open routes like GET /api/v1/repos. Log the real error server-side (chain via {e:#}) and return opaque generic messages; connection-level failures still map to 503 db_unavailable. --- crates/gitlawb-node/src/error.rs | 107 +++++++++++++++++++++++++++++-- 1 file changed, 101 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index ee890c50..91c7891c 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -72,6 +72,14 @@ pub enum AppError { pub const DB_UNAVAILABLE_CODE: &str = "db_unavailable"; pub const DB_UNAVAILABLE_MESSAGE: &str = "database is temporarily unavailable"; +/// Generic client-facing message for `AppError::Internal`. The real error is +/// logged server-side; never put sqlx/anyhow detail in the HTTP body (#226). +pub const INTERNAL_ERROR_MESSAGE: &str = "an internal error occurred"; + +/// Generic client-facing message for non-unavailable `AppError::Db`. Query / +/// schema errors stay in logs; the HTTP body must not leak them (#226). +pub const DB_ERROR_MESSAGE: &str = "a database error occurred"; + /// Connection-level sqlx failures that mean the database is unreachable right /// now (retryable, 503), as opposed to server-reported query errors. fn db_unavailable(e: &sqlx::Error) -> bool { @@ -168,12 +176,29 @@ impl IntoResponse for AppError { AppError::Overloaded(msg) => { (StatusCode::SERVICE_UNAVAILABLE, "overloaded", msg.clone()) } - AppError::Db(e) => (StatusCode::INTERNAL_SERVER_ERROR, "db_error", e.to_string()), - AppError::Internal(e) => ( - StatusCode::INTERNAL_SERVER_ERROR, - "internal_error", - e.to_string(), - ), + // Opaque body + server log: bare `?` on sqlx paths becomes `AppError::Db` + // via `From`, so this arm (not `Internal`) is the common leak for open + // routes like GET /api/v1/repos and GET /api/v1/peers (#226). + AppError::Db(e) => { + tracing::error!(error = %e, "database error"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + "db_error", + DB_ERROR_MESSAGE.into(), + ) + } + // Opaque body: handlers that map with `.map_err(AppError::Internal)` + // (e.g. GET /ipfs/{cid}) land here; other DB failures usually hit `Db`. + // Log `{e:#}` so context-wrapped anyhow chains keep the leaf cause + // (Display alone is only the outermost layer; see api/repos.rs). + AppError::Internal(e) => { + tracing::error!(error = %format!("{e:#}"), "internal error"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + "internal_error", + INTERNAL_ERROR_MESSAGE.into(), + ) + } }; let body = Json(json!({ @@ -223,4 +248,74 @@ mod tests { "1" ); } + + /// #226: raw sqlx/DB detail must never appear in the Internal 500 body. + #[tokio::test] + async fn internal_error_body_is_opaque() { + use serde_json::{json, Value}; + + let leak = "error returned from database: relation \"repos\" does not exist"; + let resp = AppError::Internal(anyhow::anyhow!("{leak}")).into_response(); + assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR); + + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("read body"); + let v: Value = serde_json::from_slice(&bytes).expect("json body"); + // Exact object: a new `detail` field with different sensitive text must + // also fail, not only a repeat of the original error string. + assert_eq!( + v, + json!({ + "error": "internal_error", + "message": INTERNAL_ERROR_MESSAGE, + }) + ); + } + + /// #226: `AppError::Db` query errors (the common `?` path) must also be opaque. + #[tokio::test] + async fn db_error_body_is_opaque() { + use serde_json::{json, Value}; + + let resp = AppError::Db(sqlx::Error::Protocol( + "error returned from database: column \"is_public\" does not exist".into(), + )) + .into_response(); + assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR); + + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("read body"); + let v: Value = serde_json::from_slice(&bytes).expect("json body"); + assert_eq!( + v, + json!({ + "error": "db_error", + "message": DB_ERROR_MESSAGE, + }) + ); + } + + /// Connection-level failures must stay 503 `db_unavailable`, not collapse + /// into the opaque 500 `db_error` arm if `db_unavailable` loses a variant. + #[tokio::test] + async fn db_pool_timeout_stays_503_unavailable() { + use serde_json::{json, Value}; + + let resp = AppError::Db(sqlx::Error::PoolTimedOut).into_response(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("read body"); + let v: Value = serde_json::from_slice(&bytes).expect("json body"); + assert_eq!( + v, + json!({ + "error": DB_UNAVAILABLE_CODE, + "message": DB_UNAVAILABLE_MESSAGE, + }) + ); + } } From 639ebaa94e94c21b4eb35e18c9c0d307523ee475 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Sat, 15 Aug 2026 08:04:34 +0600 Subject: [PATCH 20/26] fix(node): fence reconciliation pin batches against mid-batch policy changes Reviewer R1-P1 (delayed-upload race) and R1-P2 (exhausted-budget interaction) for issue-218's reconciliation sweep: - policy-epoch fence (v28 repos.policy_epoch): every visibility-rule and quarantine mutation bumps the epoch; the sweep captures it at each pin dispatch boundary and the pin loops abort the moment it moves, so a narrow landing mid-batch wins over the pre-authorized snapshot (fail closed). - encrypted seal path fenced the same way per blob; sweep acquires the pin semaphore before both public batches and the encrypted seal so the sweep cannot stack unlimited blocking pool work. - encrypt_and_pin takes git_bin + batch_budget and runs each object read under spawn_blocking with a shared read deadline via the new read_object_bounded_spawn_blocking, so a hung git reaps within budget (recovered_pins budget test). - cursor reset: run_pass fetches REPOS_PER_PASS+1 (lookahead) so a full terminal page clears the cursor instead of rescanning it forever. - gaps_filled counts unique objects across both backends so it stays countable against the union gaps_found (R2-P3). - list_pins doc reconciled with the pinata_cid-under-cid fallback it actually emits (R2-P2). - migrations v18/v19 renumbered to v26/v27 to dodge open #173's 18-25 claim. New tests: pin_new_objects_stops_mid_batch_when_policy_moves, encrypt_and_pin_stops_sealing_when_reader_removed_mid_batch, encrypt_and_pin_returns_by_budget_with_a_hung_git, and sweep_clears_cursor_on_exact_page_boundary. --- crates/gitlawb-attest/src/attestation.rs | 10 +- crates/gitlawb-core/src/identity.rs | 5 +- crates/gitlawb-node/src/api/ipfs.rs | 10 +- crates/gitlawb-node/src/api/repos.rs | 14 + crates/gitlawb-node/src/db/mod.rs | 75 ++++- crates/gitlawb-node/src/encrypted_pin.rs | 351 +++++++++++++++++++- crates/gitlawb-node/src/git/store.rs | 5 + crates/gitlawb-node/src/ipfs_pin.rs | 165 +++++++++- crates/gitlawb-node/src/main.rs | 11 +- crates/gitlawb-node/src/pinata.rs | 20 ++ crates/gitlawb-node/src/reconciliation.rs | 372 ++++++++++++++++------ 11 files changed, 917 insertions(+), 121 deletions(-) diff --git a/crates/gitlawb-attest/src/attestation.rs b/crates/gitlawb-attest/src/attestation.rs index 10a2fa34..88ab2fda 100644 --- a/crates/gitlawb-attest/src/attestation.rs +++ b/crates/gitlawb-attest/src/attestation.rs @@ -495,11 +495,17 @@ mod tests { // Public key A = identity point (0,1); signature R = identity, S = 0. // The equation `[S]B = R + [k]A` then holds for any k and any message. - let identity = [1u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let identity = [ + 1u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, + ]; let mut buf = Vec::with_capacity(34); buf.extend_from_slice(&ED25519_MULTICODEC); buf.extend_from_slice(&identity); - att.signer = format!("did:key:{}", multibase::encode(multibase::Base::Base58Btc, &buf)); + att.signer = format!( + "did:key:{}", + multibase::encode(multibase::Base::Base58Btc, &buf) + ); let mut sig = [0u8; 64]; sig[..32].copy_from_slice(&identity); att.sig = B64U.encode(sig); diff --git a/crates/gitlawb-core/src/identity.rs b/crates/gitlawb-core/src/identity.rs index 05bb9e6b..ca87f8ec 100644 --- a/crates/gitlawb-core/src/identity.rs +++ b/crates/gitlawb-core/src/identity.rs @@ -225,7 +225,10 @@ mod tests { fn verify_rejects_identity_point_forgery() { use ed25519_dalek::Verifier; // The identity point (0,1) compresses to y = 1 with sign bit 0. - let identity = [1u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let identity = [ + 1u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, + ]; let vk = VerifyingKey::from_bytes(&identity).expect("identity point is on the curve"); let mut sig_bytes = [0u8; 64]; sig_bytes[..32].copy_from_slice(&identity); diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 8a2673cd..59dce11b 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -685,11 +685,11 @@ pub async fn get_by_cid( /// when it was pinned. For Pinata-only rows (no local IPFS pin), the `cid` /// field carries `pinata_cid` so CLI consumers see a usable value. /// -/// The raw `pinata_cid` is deliberately NOT surfaced: it is node-internal state -/// (which Pinata identity a blob was uploaded with) and exposing it to -/// unauthenticated callers leaks infrastructure detail (R1-P3). Rows with -/// neither a local nor a Pinata CID are omitted so `cid` stays an -/// always-string field. +/// Rows with neither a local nor a Pinata CID are omitted so `cid` stays an +/// always-string field. The raw `pinata_cid` is not surfaced under its own +/// key, but it does appear as the `cid` value for Pinata-only rows; that is +/// the intended fallback, so the field is always a real CIDv1, never node +/// infrastructure detail beyond the CID itself. pub async fn list_pins(State(state): State) -> Result> { // Bare `?` so connection-class sqlx failures downcast to `AppError::Db` and // map to 503 `db_unavailable` (not 500 via `.map_err(AppError::Internal)`) (#251). diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 2ba4591f..cb0247db 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1170,6 +1170,10 @@ async fn pin_new_objects_gated( object_list, db, crate::ipfs_pin::PIN_BATCH_BUDGET, + // The push path derives its object list at admission and holds a write + // lease, so no sweep-style batch snapshot crosses the dispatch boundary + // (see PolicyFence's doc). + None, ) .await } @@ -1227,7 +1231,14 @@ async fn pin_and_encrypt_objects( &ctx.db, repo_id, &node_seed, + // The real git, not `ctx.git_bin`: tests point that at a fake + // walk git, and the seal reads must run the real one. + "git", + crate::ipfs_pin::PIN_BATCH_BUDGET, &recipients, + // Push path: recipients derived at admission under a write lease, + // no sweep-style snapshot to fence (see PolicyFence's doc). + None, ) .await; @@ -2434,6 +2445,9 @@ async fn post_receive_replication_tail( object_list, &db_clone, crate::ipfs_pin::PIN_BATCH_BUDGET, + // Push path: no sweep-style batch snapshot to fence (see + // PolicyFence's doc). + None, ) .await, ) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 489642bd..0e150381 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -916,11 +916,15 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE sync_queue ADD COLUMN IF NOT EXISTS attempted_at TEXT", ], }, - // 18 clears the range #135/#173 claimed (13/14) and #253 (16), so the - // backfill below cannot collide with any branch in flight. 18/19 are the - // next free integers after the merged max (17). + // Renumbered to 26/27 (was 18/19): open #173 claims the whole 18–25 range + // (pinned_cids_cid_index at 18 through its tail at 25), and the runner keys + // the applied set on the integer alone, so a version #173 also claims is + // skipped in full on whichever side merges second — no error and a silently + // absent column. main's merged max is 17, so 26+ is clear while #173 is open. + // The reservation comment above ("./17 clears both") predates #173's rebase + // onto 18–25 and is superseded by this renumber. Migration { - version: 18, + version: 26, name: "pinned_cids_clear_legacy_equal_cid", stmts: &[ // R2-P2 provenance fix: v12 allowed cid = pinata_cid as a fallback for @@ -938,7 +942,7 @@ const MIGRATIONS: &[Migration] = &[ ], }, Migration { - version: 19, + version: 27, name: "node_state", stmts: &[ // R2-P1 cursor persistence: the reconciliation sweep's keyset cursor @@ -953,6 +957,19 @@ const MIGRATIONS: &[Migration] = &[ )"#, ], }, + // v28: per-repo visibility-policy epoch (R1-P1). Every visibility mutation + // (rule set/removed, quarantine flip) bumps it, so the reconciliation sweep + // can fence a pin batch against a narrow that lands mid-batch: the pin loop + // holds the epoch captured at its dispatch boundary and aborts the remaining + // objects the moment it moves, instead of completing a pre-authorized list + // that a concurrent rule change already invalidated. + Migration { + version: 28, + name: "repos_policy_epoch", + stmts: &[ + "ALTER TABLE repos ADD COLUMN IF NOT EXISTS policy_epoch BIGINT NOT NULL DEFAULT 0", + ], + }, ]; // ── Repos ───────────────────────────────────────────────────────────────────── @@ -1530,7 +1547,11 @@ impl Db { .bind(repo_id) .execute(&self.pool) .await?; - Ok(result.rows_affected()) + let affected = result.rows_affected(); + if affected > 0 { + self.bump_repo_policy_epoch(repo_id).await?; + } + Ok(affected) } /// Repo ids currently quarantined, for operator review. Allowed dead outside @@ -2755,7 +2776,7 @@ impl Db { } /// Given a list of sha256_hex values, returns the subset that have a real - /// local IPFS CID (`cid IS NOT NULL`; after migration v18 provenance is + /// local IPFS CID (`cid IS NOT NULL`; after migration v26 provenance is /// recorded, never inferred from CID inequality). Used by the reconciliation /// sweep to skip IPFS-complete objects. /// @@ -2786,7 +2807,7 @@ impl Db { /// `cid` is left NULL for new rows so that `has_ipfs_cid` (which checks /// `cid IS NOT NULL`) correctly distinguishes local IPFS state from /// Pinata-only state. A legacy row holding `cid = pinata_cid` fallback is - /// cleared here as well (belt-and-suspenders alongside migration v18): the + /// cleared here as well (belt-and-suspenders alongside migration v26): the /// Pinata upload proves nothing about local IPFS state, so the old fallback /// value must not be misread as a local pin. pub async fn record_pinata_cid(&self, sha256_hex: &str, pinata_cid: &str) -> Result<()> { @@ -3440,6 +3461,7 @@ impl Db { .bind(&now) .execute(&self.pool) .await?; + self.bump_repo_policy_epoch(repo_id).await?; Ok(()) } @@ -3449,6 +3471,29 @@ impl Db { .bind(path_glob) .execute(&self.pool) .await?; + self.bump_repo_policy_epoch(repo_id).await?; + Ok(()) + } + + /// Current visibility-policy epoch for a repo (0 for a repo with no entry). + /// The epoch is bumped by every rule or quarantine mutation, so a value that + /// changes between two reads proves a policy change happened in between. + pub async fn repo_policy_epoch(&self, repo_id: &str) -> Result { + let row = sqlx::query("SELECT policy_epoch FROM repos WHERE id = $1") + .bind(repo_id) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.get::("policy_epoch")).unwrap_or(0)) + } + + /// Bump a repo's visibility-policy epoch by one. Callers are every mutation + /// that can change what may be publicly pinned: visibility rule writes and + /// the quarantine flag. A missing repo is not an error (0 rows affected). + async fn bump_repo_policy_epoch(&self, repo_id: &str) -> Result<()> { + sqlx::query("UPDATE repos SET policy_epoch = policy_epoch + 1 WHERE id = $1") + .bind(repo_id) + .execute(&self.pool) + .await?; Ok(()) } @@ -4364,7 +4409,7 @@ mod migration_tests { /// (2) cid IS NOT NULL, cid != pinata_cid → has_ipfs = true /// (3) cid IS NOT NULL, cid = pinata_cid (legacy) → has_ipfs = false /// - /// Legacy row (3) stops being a special case because migration v18 clears + /// Legacy row (3) stops being a special case because migration v26 clears /// `cid = pinata_cid` back to NULL, so `has_ipfs_cid` reduces to the plain /// `cid IS NOT NULL` predicate (provenance recorded, never inferred). /// @@ -4501,17 +4546,17 @@ mod migration_tests { db.migrate().await.unwrap(); } - /// Migration v18 clears legacy rows where cid was set to pinata_cid as a + /// Migration v26 clears legacy rows where cid was set to pinata_cid as a /// fallback, so `has_ipfs_cid` no longer has to infer provenance from CID /// inequality (R2-P2). Rows where the CIDs genuinely differ are untouched. #[sqlx::test] - async fn migration_v18_clears_legacy_equal_cid(pool: sqlx::PgPool) { + async fn migration_v26_clears_legacy_equal_cid(pool: sqlx::PgPool) { let db = super::Db::for_testing(pool); db.migrate().await.unwrap(); let now = "2026-07-01T12:00:00Z"; // Seed one legacy equal-cid row and one distinct-cid row, then mark - // v18 (and v19, applied after it) as not yet run so re-running + // v26 (and v27, applied after it) as not yet run so re-running // migrate() exercises the backfill in isolation. sqlx::query( "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) @@ -4522,7 +4567,7 @@ mod migration_tests { .execute(&db.pool) .await .unwrap(); - sqlx::query("DELETE FROM schema_migrations WHERE version >= 18") + sqlx::query("DELETE FROM schema_migrations WHERE version >= 26") .execute(&db.pool) .await .unwrap(); @@ -4532,7 +4577,7 @@ mod migration_tests { // Backfilled row now has no local CID; distinct row is untouched. assert!( !db.has_ipfs_cid("sha_equal").await.unwrap(), - "legacy equal-cid row must be cleared to NULL by v18" + "legacy equal-cid row must be cleared to NULL by v26" ); assert!( db.has_ipfs_cid("sha_distinct").await.unwrap(), @@ -4541,7 +4586,7 @@ mod migration_tests { assert!(db.has_pinata_cid("sha_equal").await.unwrap()); } - /// Migration v19 creates the node_state key/value table and the get/set + /// Migration v27 creates the node_state key/value table and the get/set /// helpers round-trip through it (used by the sweep cursor persistence). #[sqlx::test] async fn node_state_roundtrip_and_delete(pool: sqlx::PgPool) { diff --git a/crates/gitlawb-node/src/encrypted_pin.rs b/crates/gitlawb-node/src/encrypted_pin.rs index 19b80651..9445e9cb 100644 --- a/crates/gitlawb-node/src/encrypted_pin.rs +++ b/crates/gitlawb-node/src/encrypted_pin.rs @@ -6,6 +6,7 @@ use std::collections::{BTreeSet, HashMap}; use std::path::Path; use std::str::FromStr; +use std::time::Duration; use ed25519_dalek::VerifyingKey; use gitlawb_core::did::Did; @@ -106,17 +107,44 @@ fn plan_seal(node_seed: &[u8; 32], dids: &BTreeSet, stored_tag: Option<& /// `node_seed` keys the opaque recipients tag. Returns `(oid, cid)` for each blob /// actually sealed and recorded this call (the per-push delta), used by Option B3 /// to anchor a manifest. Recipient identities are never stored or returned. +/// +/// Nine args (the fence joins the seal's eight) but grouping them would churn +/// both callers and the race/hung-git tests for no behavioral gain. +#[allow(clippy::too_many_arguments)] pub async fn encrypt_and_pin( ipfs_api: &str, repo_path: &Path, db: &Db, repo_id: &str, node_seed: &[u8; 32], + git_bin: &str, + batch_budget: Duration, recipients: &HashMap>, + fence: Option<&crate::ipfs_pin::PolicyFence>, ) -> Vec<(String, String)> { let mut sealed = Vec::new(); let mut skipped_unresolvable = 0usize; + // One shared read deadline for the whole batch, like `pin_new_objects`: a + // hung git child is watchdog-reaped at this bound, so the outer + // `PIN_PHASE_DEADLINE` timeout cannot be held open by a blocking read + // (R1-P2). Each read runs under `spawn_blocking` — it is synchronous child + // spawn + pipe drain + watchdog join. + let read_deadline = std::time::Instant::now() + batch_budget; for (oid, dids) in recipients { + // Policy fence (R1-P1): the recipients snapshot was derived before the + // long withheld-blob walk; if a visibility rule moved while that walk + // ran (a reader added or removed), stop sealing instead of pinning to a + // stale recipient set. Checked FIRST so a changed policy costs nothing. + if let Some(f) = fence { + if !f.is_current().await { + tracing::warn!( + repo = %f.repo_id(), + oid = %oid, + "visibility policy changed after the recipients snapshot; stopping the seal loop" + ); + break; + } + } // A DB read failure is not a cache miss: re-sealing here would do an // avoidable IPFS write during a partial outage. Skip and retry next push. let stored_tag = match db.encrypted_blob_recipients_tag(repo_id, oid).await { @@ -152,7 +180,9 @@ pub async fn encrypt_and_pin( } SealPlan::Seal { keys, tag } => (keys, tag), }; - let data = match crate::git::store::read_object(repo_path, oid) { + let data = match read_object_bounded_spawn_blocking(git_bin, repo_path, oid, read_deadline) + .await + { Ok(Some((_t, bytes))) => bytes, Ok(None) => { tracing::warn!(oid = %oid, "git object not found; skipping encrypted pin"); @@ -201,10 +231,33 @@ pub async fn encrypt_and_pin( sealed } +/// Bounded, reaped git object read for the seal loop, run off the async thread: +/// `read_object_bounded` is synchronous child spawn + pipe drain + watchdog +/// join, so blocking the runtime task on it would let a hung git hold a worker +/// thread (R1-P2). The `deadline` is the batch's shared read deadline; a child +/// still alive at it is SIGTERM/SIGKILL group-reaped by the watchdog. +async fn read_object_bounded_spawn_blocking( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + deadline: std::time::Instant, +) -> anyhow::Result)>> { + let git_bin = git_bin.to_string(); + let repo_path = repo_path.to_path_buf(); + let sha256_hex = sha256_hex.to_string(); + tokio::task::spawn_blocking(move || { + crate::git::store::read_object_bounded(&git_bin, &repo_path, &sha256_hex, deadline) + .map_err(anyhow::Error::from) + }) + .await + .map_err(|e| anyhow::anyhow!("read_object spawn_blocking join failed: {e}"))? +} + #[cfg(test)] mod tests { use super::*; use ed25519_dalek::SigningKey; + use std::time::Duration; fn did_key(seed: u8) -> String { let vk = SigningKey::from_bytes(&[seed; 32]).verifying_key(); @@ -359,4 +412,300 @@ mod tests { other => panic!("changed recipient set must re-seal; got {other:?}"), } } + + /// A reader removed mid-seal must stop the seal loop (R1-P1 "race test for + /// reader removal"): `encrypt_and_pin` re-checks the policy fence before + /// each blob, so a `remove_visibility_rule` landing while the first seal is + /// in flight aborts before a later blob is pinned to a stale recipient set. + #[sqlx::test] + async fn encrypt_and_pin_stops_sealing_when_reader_removed_mid_batch(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("seal-race.git"); + + // Three loose blobs, each withheld (path-scoped deny exists so the sweep + // would have derived recipients for them). + let oids: Vec = { + crate::git::store::init_bare(&repo_path).expect("init bare repo"); + (0..3) + .map(|i| { + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(&repo_path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + child + .stdin + .as_mut() + .expect("stdin") + .write_all(format!("secret blob {i}\n").as_bytes()) + .expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }) + .collect() + }; + + // A real repos row so the fence has an epoch and a reader can be removed. + let now = chrono::Utc::now(); + let repo_id = uuid::Uuid::new_v4().to_string(); + db.create_repo(&crate::db::RepoRecord { + id: repo_id.clone(), + name: "seal-race-repo".into(), + owner_did: "did:key:zSealRaceOwner".into(), + description: None, + is_public: true, + default_branch: "main".into(), + created_at: now, + updated_at: now, + disk_path: repo_path.display().to_string(), + forked_from: None, + machine_id: None, + }) + .await + .expect("create repo"); + // A rule whose removal is the "reader removed" mutation: one reader per + // blob, all under the same path glob. + let reader = did_key(1); + db.set_visibility_rule( + &repo_id, + "**/secret/*", + crate::db::VisibilityMode::B, + std::slice::from_ref(&reader), + "did:key:zSealRaceOwner", + ) + .await + .expect("set rule"); + + // IPFS endpoint that delays the FIRST add 2s so the removal lands while + // that seal is in flight, then answers immediately. + let endpoint = delaying_cid_endpoint(vec![Duration::from_secs(2)]).await; + + let recipients: HashMap> = oids + .iter() + .cloned() + .map(|oid| { + let mut s = BTreeSet::new(); + s.insert(reader.clone()); + (oid, s) + }) + .collect(); + + let fence = crate::ipfs_pin::PolicyFence::capture(&db, &repo_id) + .await + .expect("fence captures"); + + let sealed = tokio::time::timeout(Duration::from_secs(30), async { + let seal_db = db.clone(); + let seal_repo = repo_path.clone(); + let seal_endpoint = endpoint.clone(); + let seal_repo_id = repo_id.clone(); + let handle = tokio::spawn(async move { + encrypt_and_pin( + &seal_endpoint, + &seal_repo, + &seal_db, + &seal_repo_id, + &SEED, + "git", + Duration::from_secs(60), + &recipients, + Some(&fence), + ) + .await + }); + // Let the first add start (endpoint sleeps 2s), then remove the + // reader so the fence is stale before the loop checks again. + tokio::time::sleep(Duration::from_millis(300)).await; + db.remove_visibility_rule(&repo_id, "**/secret/*") + .await + .expect("remove rule"); + handle.await.expect("seal task") + }) + .await + .expect("wedge guard: the fence abort must not take 30s"); + + assert!( + sealed.len() < oids.len(), + "a reader removal landing mid-batch must abort before every blob is sealed: {}", + sealed.len() + ); + assert!( + !sealed.is_empty(), + "at least the blob already in flight before the removal completes" + ); + } + + /// A hung git must not hold the seal loop past its read budget (R1-P2): the + /// git read runs under `spawn_blocking` against `read_object_bounded`, so + /// the watchdog reaps a wedged child at the batch deadline and the loop + /// keeps its shape instead of blocking a runtime worker indefinitely. + #[cfg(unix)] + #[sqlx::test] + async fn encrypt_and_pin_returns_by_budget_with_a_hung_git(pool: sqlx::PgPool) { + use std::os::unix::fs::PermissionsExt; + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("seal-hung.git"); + let oids: Vec = { + crate::git::store::init_bare(&repo_path).expect("init bare repo"); + (0..2) + .map(|i| { + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(&repo_path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + child + .stdin + .as_mut() + .expect("stdin") + .write_all(format!("secret blob {i}\n").as_bytes()) + .expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }) + .collect() + }; + + // A git that wedges forever, ignoring SIGTERM, so only the watchdog's + // SIGKILL can reap it. + let fake = tmp.path().join("hanging-git"); + std::fs::write(&fake, "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nsleep 30\n").unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + + let now = chrono::Utc::now(); + let repo_id = uuid::Uuid::new_v4().to_string(); + db.create_repo(&crate::db::RepoRecord { + id: repo_id.clone(), + name: "seal-hung-repo".into(), + owner_did: "did:key:zSealHungOwner".into(), + description: None, + is_public: true, + default_branch: "main".into(), + created_at: now, + updated_at: now, + disk_path: repo_path.display().to_string(), + forked_from: None, + machine_id: None, + }) + .await + .expect("create repo"); + db.set_visibility_rule( + &repo_id, + "**/secret/*", + crate::db::VisibilityMode::B, + &[did_key(1)], + "did:key:zSealHungOwner", + ) + .await + .expect("set rule"); + + let recipients: HashMap> = oids + .iter() + .cloned() + .map(|oid| { + let mut s = BTreeSet::new(); + s.insert(did_key(1)); + (oid, s) + }) + .collect(); + + // Unreachable endpoint: even if a read somehow succeeded, the pin would + // fail; the read itself is the thing under test. + let started = std::time::Instant::now(); + let sealed = tokio::time::timeout( + Duration::from_secs(60), + encrypt_and_pin( + "http://127.0.0.1:9", + &repo_path, + &db, + &repo_id, + &SEED, + fake.to_str().unwrap(), + Duration::from_secs(2), + &recipients, + None, + ), + ) + .await + .expect("a hung git must not hold the seal past the outer wedge guard"); + + let elapsed = started.elapsed(); + assert!( + elapsed < Duration::from_secs(10), + "a hung git must be watchdog-reaped inside the read budget, not block the loop for ~10s+ (took {elapsed:?})" + ); + assert!( + sealed.is_empty(), + "with a hung git no blob can be read, so nothing may be reported sealed" + ); + } + + /// Local TCP endpoint that answers `{ "Hash": "QmMock" }` after an optional + /// per-request delay, so a seal can be made to straddle a policy mutation. + async fn delaying_cid_endpoint(delays: Vec) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let mut seen = 0usize; + while let Ok((mut sock, _)) = listener.accept().await { + let delay = *delays + .get(seen) + .or_else(|| delays.last()) + .unwrap_or(&Duration::ZERO); + seen += 1; + tokio::spawn(async move { + let mut acc = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + let n = match sock.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + acc.extend_from_slice(&buf[..n]); + if let Some(hdr_end) = + acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) + { + let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); + let len: usize = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if acc.len() >= hdr_end + len { + break; + } + } + } + tokio::time::sleep(delay).await; + let body = br#"{"Hash":"QmSealRaceMockCid"}"#; + let _ = sock + .write_all( + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len()) + .as_bytes(), + ) + .await; + let _ = sock.write_all(body).await; + let _ = sock.flush().await; + }); + } + }); + endpoint + } } diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 80b63230..9c673341 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -273,6 +273,9 @@ pub struct TreeEntry { /// /// Get just the object type. Returns `None` if the object doesn't exist; a /// probe that could not examine the object store is `Err`, never `None`. +// Kept for tests and the bounded variants' docs; the async serve/seal paths use +// the `_bounded` forms. +#[allow(dead_code)] pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> { let type_output = Command::new("git") .args(["cat-file", "-t", sha256_hex]) @@ -305,6 +308,7 @@ pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> } /// Read an object's content if its type is already known. +#[allow(dead_code)] pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) -> Result> { let content_output = Command::new("git") .args(["cat-file", obj_type, sha256_hex]) @@ -678,6 +682,7 @@ pub fn read_object_bounded( /// `gitlawb_core::cid::Cid::from_git_object_bytes`. /// /// Returns `None` if the object does not exist in this repo. +#[allow(dead_code)] pub fn read_object(repo_path: &Path, sha256_hex: &str) -> Result)>> { let obj_type = match object_type(repo_path, sha256_hex)? { Some(t) => t, diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 60de786e..7a90badf 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -26,6 +26,59 @@ use std::time::{Duration, Instant}; /// have to be documented, validated, and kept meaningful. pub const PIN_BATCH_BUDGET: Duration = Duration::from_secs(120); +/// A captured per-repo visibility-policy epoch that fences a pin batch. +/// +/// The reconciliation sweep reads the epoch immediately before dispatching a +/// pin loop and passes a fence in; the loop re-reads the epoch before every +/// upload and aborts the batch the moment it moves. A visibility narrow that +/// lands mid-batch (a rule made private, a repo quarantined) must not let the +/// remaining pre-authorized objects still go to a public content-addressed +/// backend — the narrow is a policy change, and dispatching against the stale +/// snapshot is the exact irreversible-publication class this fence exists for +/// (R1-P1). `None` (the push path) means "no fence": the push derives its own +/// object list at admission and holds a write lease, so no sweep-style batch +/// snapshot crosses the dispatch boundary. +#[derive(Clone)] +pub struct PolicyFence { + db: crate::db::Db, + repo_id: String, + epoch: i64, +} + +impl PolicyFence { + /// Capture the current policy epoch for `repo_id`. A read failure is a + /// skip, not a retry-with-zero: the caller must not dispatch a batch it + /// cannot fence (fail closed on a stale allow). + pub async fn capture(db: &crate::db::Db, repo_id: &str) -> Option { + match db.repo_policy_epoch(repo_id).await { + Ok(epoch) => Some(PolicyFence { + db: db.clone(), + repo_id: repo_id.to_string(), + epoch, + }), + Err(e) => { + tracing::warn!(repo = %repo_id, err = %e, "policy-epoch read failed; not fencing pin batch"); + None + } + } + } + + /// Whether the repo's policy epoch is unchanged since capture. A read + /// failure is treated as "changed": never dispatch on a policy we cannot + /// prove current. + pub async fn is_current(&self) -> bool { + match self.db.repo_policy_epoch(&self.repo_id).await { + Ok(epoch) => epoch == self.epoch, + Err(_) => false, + } + } + + /// The repo this fence guards, for log correlation. + pub fn repo_id(&self) -> &str { + &self.repo_id + } +} + /// The smallest remainder worth starting a bounded git read (or an add) with. /// /// A 1ms remainder otherwise buys a child spawned already past its deadline, which @@ -256,6 +309,7 @@ pub async fn pin_new_objects( object_list: Vec, db: &crate::db::Db, batch_budget: Duration, + fence: Option<&PolicyFence>, ) -> Vec<(String, String)> { if ipfs_api.is_empty() { return vec![]; @@ -266,6 +320,19 @@ pub async fn pin_new_objects( let mut pinned = Vec::new(); for (attempted, sha) in object_list.into_iter().enumerate() { + // Policy fence (R1-P1): a visibility narrow that lands after the caller + // built this batch must abort it before the next irreversible upload. + // Checked FIRST so a changed policy costs nothing beyond the read. + if let Some(f) = fence { + if !f.is_current().await { + tracing::warn!( + repo = %f.repo_id, + unattempted = total - attempted, + "visibility policy changed mid-batch; stopping the pin loop" + ); + break; + } + } // Top of the iteration, before any of this object's work: an object is // never started with a remainder too small to cover a bounded read's // teardown. Consumed as a guard only: the read below runs against the @@ -275,7 +342,7 @@ pub async fn pin_new_objects( break; } // Skip if already pinned to local IPFS. This checks the real `cid` - // column, NOT whether any row exists: after migration v18 cleared the + // column, NOT whether any row exists: after migration v26 cleared the // legacy `cid = pinata_cid` fallback, a Pinata-only row has `cid = NULL` // and must be (re)pinned to IPFS, so the skip predicate is `cid IS NOT // NULL` (`has_ipfs_cid`), never `is_pinned` (which counts Pinata-only @@ -667,6 +734,7 @@ mod tests { oids, &db, Duration::from_millis(5500), + None, ), ) .await @@ -732,6 +800,7 @@ mod tests { oids, &db, Duration::from_secs(90), + None, ), ) .await @@ -768,6 +837,7 @@ mod tests { oids, &db, Duration::from_secs(60), + None, ), ) .await @@ -863,6 +933,7 @@ mod tests { oids, &db, Duration::from_secs(2), + None, ), ) .await @@ -948,6 +1019,7 @@ mod tests { oids, &db, Duration::from_millis(1500), + None, ), ) .await @@ -1019,6 +1091,7 @@ mod tests { oids.clone(), &db, Duration::from_secs(60), + None, ), ) .await @@ -1091,6 +1164,7 @@ mod tests { oids.clone(), &db, Duration::from_secs(60), + None, ), ) .await @@ -1157,6 +1231,7 @@ mod tests { oids.clone(), &db, Duration::from_secs(60), + None, ), ) .await @@ -1173,4 +1248,92 @@ mod tests { "one corrupt object must cost only itself: the other four must still pin" ); } + + /// The policy fence must stop a batch whose policy moved after it was + /// built (R1-P1 "delayed-upload race"). The first add is delayed long + /// enough for a quarantine to land mid-batch; the loop's per-iteration + /// fence check must abort before any later object is uploaded, so the + /// narrow wins even though the batch was built under the old allow. + #[sqlx::test] + async fn pin_new_objects_stops_mid_batch_when_policy_moves(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("race.git"); + let oids = seed_loose_blobs(&repo_path, 4); + let endpoint = delaying_endpoint(vec![Duration::from_secs(2)]).await; + + // A real repos row, so the fence has an epoch to observe and quarantine + // can bump it. + let now = chrono::Utc::now(); + let repo_id = uuid::Uuid::new_v4().to_string(); + db.create_repo(&crate::db::RepoRecord { + id: repo_id.clone(), + name: "race-repo".into(), + owner_did: "did:key:zRaceOwner".into(), + description: None, + is_public: true, + default_branch: "main".into(), + created_at: now, + updated_at: now, + disk_path: repo_path.display().to_string(), + forked_from: None, + machine_id: None, + }) + .await + .expect("create repo"); + + let fence = PolicyFence::capture(&db, &repo_id) + .await + .expect("fence captures before any mutation"); + assert_eq!( + db.repo_policy_epoch(&repo_id).await.unwrap(), + 0, + "a fresh repo starts at epoch 0" + ); + + // Run the batch; quarantine lands while the first add (2s) is in + // flight, then the next iteration's fence check must abort. + let (logs, _guard) = capture_logs(); + let pinned = tokio::time::timeout(Duration::from_secs(30), async { + let pin_db = db.clone(); + let pin_endpoint = endpoint.clone(); + let pin_repo_path = repo_path.clone(); + let handle = tokio::spawn(async move { + pin_new_objects( + &pin_endpoint, + &pin_repo_path, + "git", + oids, + &pin_db, + Duration::from_secs(60), + Some(&fence), + ) + .await + }); + // Let the first add start (endpoint sleeps 2s), then move the + // policy so the fence is stale before the loop checks again. + tokio::time::sleep(Duration::from_millis(300)).await; + db.set_repo_quarantine(&repo_id, true) + .await + .expect("quarantine"); + handle.await.expect("pin task") + }) + .await + .expect("wedge guard: the fence abort must not take 30s"); + + assert!( + pinned.len() < 4, + "a quarantine landing mid-batch must abort before every object uploads, not pin all four: {}", + pinned.len() + ); + assert!( + !pinned.is_empty(), + "at least the object already in flight before the quarantine completes" + ); + assert!( + logs.text().contains("visibility policy changed mid-batch"), + "the abort must be logged so operators can see why the batch stopped" + ); + } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 6d642361..1eb34dda 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -581,8 +581,17 @@ async fn main() -> Result<()> { let http_client = Arc::clone(&state.http_client); let node_keypair = Arc::clone(&state.node_keypair); let node_did = state.node_did.clone(); + let pin_sem = Arc::clone(&state.pin_semaphore); let shutdown_rx = state.subscribe_shutdown(); - if reconciliation::spawn(db, config, http_client, node_keypair, node_did, shutdown_rx) { + if reconciliation::spawn( + db, + config, + http_client, + node_keypair, + node_did, + pin_sem, + shutdown_rx, + ) { info!("reconciliation sweep worker started"); } } diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 90942348..8a4478a2 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -122,6 +122,7 @@ pub async fn pin_new_objects( object_list: Vec, db: &crate::db::Db, batch_budget: Duration, + fence: Option<&crate::ipfs_pin::PolicyFence>, ) -> Vec<(String, String)> { if jwt.is_empty() { return vec![]; @@ -132,6 +133,18 @@ pub async fn pin_new_objects( let mut pinned = Vec::new(); for (attempted, sha) in object_list.into_iter().enumerate() { + // Policy fence (R1-P1): a visibility narrow that lands after the caller + // built this batch must abort it before the next irreversible upload. + if let Some(f) = fence { + if !f.is_current().await { + tracing::warn!( + repo = %f.repo_id(), + unattempted = total - attempted, + "visibility policy changed mid-batch; stopping the Pinata pin loop" + ); + break; + } + } // Top of the iteration, before any of this object's work: an object is never // started with a remainder too small to cover a bounded read's teardown. The // gate is shared with the IPFS loop so the two cannot drift apart in how they @@ -441,6 +454,7 @@ mod tests { oids, &db, Duration::from_millis(5500), + None, ), ) .await @@ -530,6 +544,7 @@ mod tests { oids, &db, Duration::from_secs(2), + None, ), ) .await @@ -658,6 +673,7 @@ mod tests { oids, &db, Duration::from_secs(60), + None, ), ) .await @@ -730,6 +746,7 @@ mod tests { oids, &db, Duration::from_secs(60), + None, ), ) .await @@ -779,6 +796,7 @@ mod tests { oids.clone(), &db, Duration::from_secs(60), + None, ), ) .await @@ -806,6 +824,7 @@ mod tests { oids, &db, Duration::from_secs(60), + None, ), ) .await @@ -857,6 +876,7 @@ mod tests { oids, &db, Duration::from_secs(60), + None, ), ) .await diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs index b1682590..4419f2af 100644 --- a/crates/gitlawb-node/src/reconciliation.rs +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -25,7 +25,19 @@ const REPO_SCAN_DEADLINE: Duration = Duration::from_secs(300); /// Per-repo deadline for the pinning phase (IPFS + Pinata uploads). An /// unavailable backend that stalls per-object must not hold the sweep for -/// the entire backlog; this bounds the total wall time per repo per pass. +/// the entire backlog; this bounds the wall time of each pinning PHASE. +/// +/// The phases do NOT share one budget (R2-P3): the scan, the pin-boundary +/// authorization re-derivation (`pin_authz_deadline`), the withheld-blob walk, +/// and each pin/seal phase each get their own `REPO_SCAN_DEADLINE` / +/// `PIN_PHASE_DEADLINE`. A repo's worst case is therefore ADDITIVE, up to +/// ~25min in pathological conditions (scan 5m + authz re-derivation 5m + +/// withheld walk 5m + public pin 5m + encrypted seal 5m), not bounded at a +/// single deadline. That is a deliberate trade: starving a later phase of the +/// budget the scan consumed would silently disable the authorization check or +/// the recovery-copy seal for exactly the large repos the sweep exists for. +/// The sweep runs hourly and each phase is still individually bounded, so a +/// pathological repo delays other repos by at most that phase, not the hour. const PIN_PHASE_DEADLINE: Duration = Duration::from_secs(300); /// node_state key under which the sweep's keyset cursor is persisted across @@ -51,6 +63,7 @@ pub fn spawn( http_client: Arc, node_keypair: Arc, node_did: gitlawb_core::did::Did, + pin_sem: Arc, mut shutdown_rx: watch::Receiver, ) -> bool { if !should_spawn(&config) { @@ -81,6 +94,7 @@ pub fn spawn( &http_client, &node_seed, &node_did, + &pin_sem, &mut cursor, &mut shutdown_rx, ) @@ -126,11 +140,13 @@ pub fn spawn( /// runs against rules re-fetched after the git scan, so a narrowing made /// mid-scan is honored before anything is pinned. /// -/// The full scan and this re-filter carry the same absolute `deadline` so the -/// total blocking time per repo stays bounded: `run_pass` computes one deadline -/// and hands it to both stages, and each stage runs against the remaining -/// budget rather than a fresh full timeout (the two stages share the document -/// per-repo cap, not two independent ones). +/// The caller hands an absolute `deadline`; the whole re-derivation +/// (replicable_blob_set_bounded + all_blob_oids) runs against the remaining +/// budget rather than granting each git child a fresh timeout. The mid-scan +/// re-filter shares the scan deadline with the full scan (one per-repo read +/// cap), but the pin-boundary re-derivations use a separate `pin_authz_deadline` +/// so a scan that exhausts its own budget cannot disable the +/// authorization-at-dispatch recheck. async fn refilter_public_objects( disk: &std::path::Path, rules: &[crate::db::VisibilityRule], @@ -265,12 +281,18 @@ fn cap_missing(v: Vec, repo_slug: &str, backend: &str) -> Vec { /// and hard skips excluded, and the loop stops counting the moment a shutdown /// signal breaks the batch), so the returned value never overreports work that /// a mid-pass shutdown prevented (R1-P3). +/// +/// Eight args (the pin semaphore joins the original seven) but grouping them +/// would churn every test caller for no behavioral gain; the pins each arg +/// names are independently documented at their use. +#[allow(clippy::too_many_arguments)] async fn run_pass( db: &Db, config: &Config, http_client: &reqwest::Client, node_seed: &[u8; 32], node_did: &gitlawb_core::did::Did, + pin_sem: &Arc, cursor: &mut Option, shutdown_rx: &mut watch::Receiver, ) -> anyhow::Result<(usize, usize, usize)> { @@ -278,9 +300,17 @@ async fn run_pass( // robust against insertions, deletions, or updated_at shifts. The LIMIT // is pushed into the SQL query so the hourly pass does not allocate, // transfer, or deduplicate every repo on every sweep. - let batch = db - .list_all_repos_deduped_stable(cursor.as_deref(), REPOS_PER_PASS as i64) + // + // Fetch one EXTRA row as a lookahead (R1-P2): `batch.len() < REPOS_PER_PASS` + // is a wrong "final page" proxy when the key space ends on an exact multiple + // of the page size — that batch LOOKS full, yet no row follows. With a + // lookahead row present, the batch is full for real (more remain); without + // it, the batch is the terminal page even at exactly REPOS_PER_PASS rows. + let fetched = db + .list_all_repos_deduped_stable(cursor.as_deref(), REPOS_PER_PASS as i64 + 1) .await?; + let has_more = fetched.len() > REPOS_PER_PASS; + let batch: Vec<_> = fetched.into_iter().take(REPOS_PER_PASS).collect(); if batch.is_empty() { // Covered everything: clear the persisted cursor so the next pass @@ -363,9 +393,13 @@ async fn run_pass( // ── Full git scan (bounded) ───────────────────────────────────── // One absolute deadline spans the whole scan AND the mandatory - // visibility re-filter below, so a repo's total blocking time stays - // bounded at REPO_SCAN_DEADLINE (they share the cap, not two - // independent ones). Each stage runs against the remaining budget. + // visibility re-filter below, so a repo's total blocking time for the + // read phase stays bounded at REPO_SCAN_DEADLINE (they share the cap, + // not two independent ones). Each stage runs against the remaining + // budget. The pin-boundary re-derivations below use a FRESH deadline + // (see `pin_authz_deadline`), so a scan that legitimately consumes its + // whole budget cannot silently disable the authorization-at-dispatch + // recheck for this repo's pinning. let scan_deadline = Instant::now() + REPO_SCAN_DEADLINE; let disk_clone = disk.clone(); let owner_clone = repo.owner_did.clone(); @@ -429,6 +463,15 @@ async fn run_pass( continue; } + // Fresh budget for the authorization-at-dispatch re-derivations (R1/R2): + // the scan may have legitimately consumed its whole `scan_deadline`, and + // reusing that deadline here would compute a zero remaining duration, + // return None, and turn an empty `to_pin` into a permanent hourly skip + // for exactly the large/slow repos the sweep exists for. This deadline is + // deliberately NOT shared with the scan: it bounds only the pin-boundary + // re-derivations (both backends, as one unit). + let pin_authz_deadline = Instant::now() + REPO_SCAN_DEADLINE; + // ── Phase 1: Public-object pinning (IPFS + Pinata) ──────────────── // Re-check quarantine AND visibility right now (fresh rules + repo row), // then re-derive the allowed set from those fresh rules so a path-scoped @@ -510,102 +553,142 @@ async fn run_pass( // irreversible, and the pin itself takes time. A path-scoped deny that // landed after the mid-scan refilter (which only checks root listability) // is honored here because the candidates are intersected with the set - // allowed under the fresh rules, not just root-gated. + // allowed under the fresh rules, not just root-gated. Each backend runs + // under a PolicyFence captured at ITS dispatch boundary, so a narrow that + // lands mid-batch aborts the remaining uploads (R1-P1). + // + // Acquire the same global pin permit the push path holds (R2-P2): the + // sweep's pin loops must not bypass `max_concurrent_pin_tasks`. Acquired + // only when there is actual pin work; the scan above holds no permit. + // The permit is held across the public pin loops AND the encrypted seal + // below (which also writes to IPFS) and dropped at the end of this repo's + // iteration. + let _pin_permit = if !ipfs_missing.is_empty() || !pinata_missing.is_empty() { + let permit = pin_sem.clone().acquire_owned().await?; + Some(permit) + } else { + None + }; + let ipfs_fence = if ipfs_enabled && !ipfs_missing.is_empty() { + crate::ipfs_pin::PolicyFence::capture(db, &repo.id).await + } else { + None + }; + let pinata_fence = if pinata_enabled && !pinata_missing.is_empty() { + crate::ipfs_pin::PolicyFence::capture(db, &repo.id).await + } else { + None + }; + let pinned_ipfs: Vec<(String, String)> = if ipfs_enabled && !ipfs_missing.is_empty() { - match recheck_public_pin(db, &repo.id, &repo_slug).await { - None => Vec::new(), - Some((fresh_repo, fresh_rules)) => { - let to_pin = match refilter_public_objects( - &disk, - &fresh_rules, - fresh_repo.is_public, - &fresh_repo.owner_did, - ipfs_missing, - scan_deadline, - ) - .await - { - Some(list) => list, - None => { - tracing::warn!(repo = %repo_slug, "IPFS pin-boundary re-derivation failed, skipping"); - Vec::new() - } - }; - if to_pin.is_empty() { - Vec::new() - } else { - match tokio::time::timeout( - PIN_PHASE_DEADLINE, - crate::ipfs_pin::pin_new_objects( - &config.ipfs_api, - &disk, - "git", - to_pin, - db, - crate::ipfs_pin::PIN_BATCH_BUDGET, - ), + match ipfs_fence { + None => { + tracing::warn!(repo = %repo_slug, "IPFS policy-epoch capture failed, skipping"); + Vec::new() + } + Some(fence) => match recheck_public_pin(db, &repo.id, &repo_slug).await { + None => Vec::new(), + Some((fresh_repo, fresh_rules)) => { + let to_pin = match refilter_public_objects( + &disk, + &fresh_rules, + fresh_repo.is_public, + &fresh_repo.owner_did, + ipfs_missing, + pin_authz_deadline, ) .await { - Ok(v) => v, - Err(_) => { - tracing::warn!(repo = %repo_slug, "IPFS pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + Some(list) => list, + None => { + tracing::warn!(repo = %repo_slug, "IPFS pin-boundary re-derivation failed, skipping"); Vec::new() } + }; + if to_pin.is_empty() { + Vec::new() + } else { + match tokio::time::timeout( + PIN_PHASE_DEADLINE, + crate::ipfs_pin::pin_new_objects( + &config.ipfs_api, + &disk, + "git", + to_pin, + db, + crate::ipfs_pin::PIN_BATCH_BUDGET, + Some(&fence), + ), + ) + .await + { + Ok(v) => v, + Err(_) => { + tracing::warn!(repo = %repo_slug, "IPFS pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + Vec::new() + } + } } } - } + }, } } else { Vec::new() }; let pinned_pinata: Vec<(String, String)> = if pinata_enabled && !pinata_missing.is_empty() { - match recheck_public_pin(db, &repo.id, &repo_slug).await { - None => Vec::new(), - Some((fresh_repo, fresh_rules)) => { - let to_pin = match refilter_public_objects( - &disk, - &fresh_rules, - fresh_repo.is_public, - &fresh_repo.owner_did, - pinata_missing, - scan_deadline, - ) - .await - { - Some(list) => list, - None => { - tracing::warn!(repo = %repo_slug, "Pinata pin-boundary re-derivation failed, skipping"); - Vec::new() - } - }; - if to_pin.is_empty() { - Vec::new() - } else { - match tokio::time::timeout( - PIN_PHASE_DEADLINE, - crate::pinata::pin_new_objects( - http_client, - &config.pinata_upload_url, - &config.pinata_jwt, - &disk, - "git", - to_pin, - db, - crate::ipfs_pin::PIN_BATCH_BUDGET, - ), + match pinata_fence { + None => { + tracing::warn!(repo = %repo_slug, "Pinata policy-epoch capture failed, skipping"); + Vec::new() + } + Some(fence) => match recheck_public_pin(db, &repo.id, &repo_slug).await { + None => Vec::new(), + Some((fresh_repo, fresh_rules)) => { + let to_pin = match refilter_public_objects( + &disk, + &fresh_rules, + fresh_repo.is_public, + &fresh_repo.owner_did, + pinata_missing, + pin_authz_deadline, ) .await { - Ok(v) => v, - Err(_) => { - tracing::warn!(repo = %repo_slug, "Pinata pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + Some(list) => list, + None => { + tracing::warn!(repo = %repo_slug, "Pinata pin-boundary re-derivation failed, skipping"); Vec::new() } + }; + if to_pin.is_empty() { + Vec::new() + } else { + match tokio::time::timeout( + PIN_PHASE_DEADLINE, + crate::pinata::pin_new_objects( + http_client, + &config.pinata_upload_url, + &config.pinata_jwt, + &disk, + "git", + to_pin, + db, + crate::ipfs_pin::PIN_BATCH_BUDGET, + Some(&fence), + ), + ) + .await + { + Ok(v) => v, + Err(_) => { + tracing::warn!(repo = %repo_slug, "Pinata pin phase timed out after {:?}", PIN_PHASE_DEADLINE); + Vec::new() + } + } } } - } + }, } } else { Vec::new() @@ -613,8 +696,13 @@ async fn run_pass( // `pin_new_objects` returns only objects whose DB record was written // (R1-P3), so a backend that uploaded bytes but failed to persist is - // not counted as "filled". - let repo_filled = pinned_ipfs.len() + pinned_pinata.len(); + // not counted as "filled". Count UNIQUE objects across both backends + // (R2-P3): `gaps_found` is the union of missing OIDs, so an object + // pinned to BOTH backends must not count twice against that union. + let mut filled_union: HashSet<&String> = HashSet::new(); + filled_union.extend(pinned_ipfs.iter().map(|(sha, _)| sha)); + filled_union.extend(pinned_pinata.iter().map(|(sha, _)| sha)); + let repo_filled = filled_union.len(); if repo_filled > 0 { total_gaps_filled += repo_filled; crate::metrics::record_reconciliation_gaps_filled(repo_filled as u64); @@ -636,6 +724,17 @@ async fn run_pass( Some(v) => v, None => continue, }; + // Fence the encrypted path from the point the recipients are derived: + // the withheld-blob walk is long, and `encrypt_and_pin` re-checks the + // epoch per blob, so a visibility rule moving mid-walk aborts the seal + // loop before a stale recipient set is pinned (R1-P1). + let enc_fence = match crate::ipfs_pin::PolicyFence::capture(db, &repo.id).await { + Some(f) => f, + None => { + tracing::warn!(repo = %repo_slug, "policy-epoch capture failed, skipping encrypted pin"); + continue; + } + }; let has_path_scoped = crate::git::visibility_pack::has_path_scoped_rule(&fresh_rules2); if has_path_scoped && ipfs_enabled { @@ -684,6 +783,11 @@ async fn run_pass( }; if !rec.is_empty() { + // The encrypted seal writes to IPFS too, so it runs under the + // same global pin permit as the public loops (R2-P2). It is + // acquired only when a seal is actually possible (the walk above + // holds no permit). + let _enc_permit = pin_sem.clone().acquire_owned().await?; // Bound the seal+pin work (R1-P2): an unavailable backend must // not hold the sweep past the pin-phase budget. let sealed = tokio::time::timeout( @@ -694,7 +798,10 @@ async fn run_pass( db, &repo.id, node_seed, + "git", + crate::ipfs_pin::PIN_BATCH_BUDGET, &rec, + Some(&enc_fence), ), ) .await; @@ -753,11 +860,12 @@ async fn run_pass( // interrupted us, leave the persisted cursor at the previous batch's end so // the next run re-walks the unprocessed tail (R2-P1, R1-P3). if batch_completed { - // A short final page means the whole key space is covered: clear the - // cursor now so the next tick starts a fresh cycle instead of burning - // one pass on an empty batch (persisting `batch_last` here would make - // the next tick scan nothing and only then reset). - if batch.len() < REPOS_PER_PASS { + // A terminal page (no lookahead row) means the whole key space is + // covered: clear the cursor now so the next tick starts a fresh cycle + // instead of burning one pass on an empty batch. The lookahead is what + // distinguishes "full because more remain" from "full because the key + // space ends on an exact page boundary" (R1-P2). + if !has_more { *cursor = None; if let Err(e) = db.set_node_state(CURSOR_KEY, None).await { tracing::warn!(err = %e, "failed to clear reconciliation sweep cursor on final page"); @@ -847,11 +955,12 @@ mod tests { let kp = std::sync::Arc::new(gitlawb_core::identity::Keypair::generate()); let node_did = kp.did(); let (_tx, rx) = watch::channel(false); + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); // spawn() should return false synchronously (no tokio::spawn) and never // await the DB. The test completes without timeout == gate is live. assert!( - !super::spawn(db, config, http, kp, node_did, rx), + !super::spawn(db, config, http, kp, node_did, pin_sem, rx), "gated spawn must report it did not start a worker" ); } @@ -870,9 +979,10 @@ mod tests { let kp = std::sync::Arc::new(gitlawb_core::identity::Keypair::generate()); let node_did = kp.did(); let (_tx, rx) = watch::channel(false); + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); assert!( - super::spawn(db, config, http, kp, node_did, rx), + super::spawn(db, config, http, kp, node_did, pin_sem, rx), "configured spawn must report it started a worker" ); } @@ -1005,6 +1115,7 @@ mod tests { let http = reqwest::Client::new(); let (_tx, mut rx) = watch::channel(false); let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); let (scanned, gaps, filled) = super::run_pass( &db, @@ -1012,6 +1123,7 @@ mod tests { &http, &node_seed, &node_did, + &pin_sem, &mut cursor, &mut rx, ) @@ -1054,6 +1166,7 @@ mod tests { &http, &node_seed, &node_did, + &pin_sem, &mut cursor, &mut rx, ) @@ -1096,6 +1209,7 @@ mod tests { let http = reqwest::Client::new(); let (_tx, mut rx) = watch::channel(false); let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); let (scanned, gaps, filled) = super::run_pass( &db, @@ -1103,6 +1217,7 @@ mod tests { &http, &node_seed, &node_did, + &pin_sem, &mut cursor, &mut rx, ) @@ -1184,6 +1299,7 @@ mod tests { let http = reqwest::Client::new(); let (_tx, mut rx) = watch::channel(false); let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); let (scanned, gaps, filled) = super::run_pass( &db, @@ -1191,6 +1307,7 @@ mod tests { &http, &node_seed, &node_did, + &pin_sem, &mut cursor, &mut rx, ) @@ -1216,4 +1333,69 @@ mod tests { "withheld blob must never be pinned to a public backend in cleartext" ); } + + /// The final-page proxy must be the lookahead, not `batch.len() < page` + /// (R1-P2): a key space ending on an exact page boundary looks "full" yet + /// has no following row, so the cursor must be CLEARED, not persisted to a + /// nonexistent next page (which would wedge the sweep into empty tail passes + /// every tick). REPOS_PER_PASS repos and nothing more must behave exactly + /// like one repo. + #[sqlx::test] + async fn sweep_clears_cursor_on_exact_page_boundary(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + // Exactly one full page of repos, each with a missing disk path (hard + // skip, never scanned, so no pinning side effects). + let n = super::REPOS_PER_PASS; + for i in 0..n { + let rec = seed_repo( + "did:key:zExactPageOwner", + &format!("exact-repo-{i:04}"), + &format!("/nonexistent/disk/path-{i:04}"), + ); + db.create_repo(&rec).await.unwrap(); + } + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + "http://127.0.0.1:1", // unreachable; must never be hit + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(2)); + + let (scanned, gaps, filled) = super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + &mut cursor, + &mut rx, + ) + .await + .unwrap(); + + assert_eq!(scanned, 0, "missing-disk rows are hard skips, not scans"); + assert_eq!(gaps, 0); + assert_eq!(filled, 0); + + let persisted = db.get_node_state(super::CURSOR_KEY).await.unwrap(); + assert!( + persisted.is_none(), + "an exact-page terminal batch must clear the cursor, not persist it \ + to a nonexistent next page (would wedge every subsequent tick)" + ); + assert!( + cursor.is_none(), + "in-memory cursor follows the persisted one" + ); + } } From b44c9519375b36fd6ea5ebb9be08ae28e8ac5da4 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Sat, 15 Aug 2026 14:30:03 +0600 Subject: [PATCH 21/26] fix(node): hold one pin permit per repo, capture encrypted fence before rules recheck Reviewer R2-P1 findings on the reconciliation sweep: - P1 permit: run_pass held the global pin permit for the whole repo iteration, then acquired a SECOND one for the same repo's seal phase. With max_concurrent_pin_tasks = 1 the sweep waited on the very permit it held, deadlocking past the guard timeout. The seal phase now reuses the permit the public phase already holds and only acquires when the public phase held none (one permit per repo, never two). - P1 ordering: the encrypted-path PolicyFence was captured AFTER recheck_public_pin's rule read, so a narrow landing between the two was baked into the recipient set while the captured epoch already reflected it and is_current stayed true for the whole seal loop. Capture now runs BEFORE the recheck, mirroring the public path. - P2 budgets: IPFS and Pinata re-derivations previously shared one pin_authz_deadline; IPFS re-derives first, so a large repo that consumed it left Pinata silently skipped every pass. Each arm now re-derives against its own fresh REPO_SCAN_DEADLINE. - P3 budget gate: the seal loop in encrypt_and_pin got the same batch_budget_gate the IPFS/Pinata loops use, so the three loops cannot drift apart in how they report a truncated batch. New test: run_pass_reuses_the_pin_permit_for_the_seal_at_pool_size_one, which reproduces the reviewer's probe (public gaps + path-scoped rule with a reader, pool size 1) and fails on the old double-acquire (verified by reverting the fix: deadlocks until the 60s test timeout). --- crates/gitlawb-node/src/encrypted_pin.rs | 18 ++- crates/gitlawb-node/src/reconciliation.rs | 149 ++++++++++++++++++---- 2 files changed, 143 insertions(+), 24 deletions(-) diff --git a/crates/gitlawb-node/src/encrypted_pin.rs b/crates/gitlawb-node/src/encrypted_pin.rs index 9445e9cb..76529d70 100644 --- a/crates/gitlawb-node/src/encrypted_pin.rs +++ b/crates/gitlawb-node/src/encrypted_pin.rs @@ -130,7 +130,23 @@ pub async fn encrypt_and_pin( // (R1-P2). Each read runs under `spawn_blocking` — it is synchronous child // spawn + pipe drain + watchdog join. let read_deadline = std::time::Instant::now() + batch_budget; - for (oid, dids) in recipients { + let total = recipients.len(); + for (attempted, (oid, dids)) in recipients.iter().enumerate() { + // Batch budget gate (R2-P3), mirroring the public pin loops: an object + // is never started with a remainder too small to cover a bounded read's + // teardown. This is consistency (the seal is bounded by the outer + // `PIN_PHASE_DEADLINE` either way), but it keeps the three loops from + // drifting apart in how they report a truncated batch. + if crate::ipfs_pin::batch_budget_gate( + "encrypted-seal", + read_deadline, + sealed.len(), + total - attempted, + ) + .is_none() + { + break; + } // Policy fence (R1-P1): the recipients snapshot was derived before the // long withheld-blob walk; if a visibility rule moved while that walk // ran (a reader added or removed), stop sealing instead of pinning to a diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs index 4419f2af..f44a3818 100644 --- a/crates/gitlawb-node/src/reconciliation.rs +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -27,10 +27,9 @@ const REPO_SCAN_DEADLINE: Duration = Duration::from_secs(300); /// unavailable backend that stalls per-object must not hold the sweep for /// the entire backlog; this bounds the wall time of each pinning PHASE. /// -/// The phases do NOT share one budget (R2-P3): the scan, the pin-boundary -/// authorization re-derivation (`pin_authz_deadline`), the withheld-blob walk, -/// and each pin/seal phase each get their own `REPO_SCAN_DEADLINE` / -/// `PIN_PHASE_DEADLINE`. A repo's worst case is therefore ADDITIVE, up to +/// The phases do NOT share one budget (R2-P3): the scan, the per-backend +/// pin-boundary authorization re-derivation, the withheld-blob walk, and each +/// pin/seal phase each get their own `REPO_SCAN_DEADLINE` / `PIN_PHASE_DEADLINE`. A repo's worst case is therefore ADDITIVE, up to /// ~25min in pathological conditions (scan 5m + authz re-derivation 5m + /// withheld walk 5m + public pin 5m + encrypted seal 5m), not bounded at a /// single deadline. That is a deliberate trade: starving a later phase of the @@ -144,9 +143,9 @@ pub fn spawn( /// (replicable_blob_set_bounded + all_blob_oids) runs against the remaining /// budget rather than granting each git child a fresh timeout. The mid-scan /// re-filter shares the scan deadline with the full scan (one per-repo read -/// cap), but the pin-boundary re-derivations use a separate `pin_authz_deadline` -/// so a scan that exhausts its own budget cannot disable the -/// authorization-at-dispatch recheck. +/// cap), but each pin-boundary re-derivation runs against its own fresh +/// `REPO_SCAN_DEADLINE` so a scan that exhausts its own budget cannot disable +/// the authorization-at-dispatch recheck. async fn refilter_public_objects( disk: &std::path::Path, rules: &[crate::db::VisibilityRule], @@ -468,9 +467,10 @@ async fn run_pass( // reusing that deadline here would compute a zero remaining duration, // return None, and turn an empty `to_pin` into a permanent hourly skip // for exactly the large/slow repos the sweep exists for. This deadline is - // deliberately NOT shared with the scan: it bounds only the pin-boundary - // re-derivations (both backends, as one unit). - let pin_authz_deadline = Instant::now() + REPO_SCAN_DEADLINE; + // deliberately NOT shared with the scan. Each backend arm re-derives + // against its OWN budget (R2-P1): IPFS re-derives first, and if both + // shared one budget a large repo that consumed it on the IPFS walk would + // leave Pinata silently skipped every pass — empty `to_pin` behind a warn. // ── Phase 1: Public-object pinning (IPFS + Pinata) ──────────────── // Re-check quarantine AND visibility right now (fresh rules + repo row), @@ -595,7 +595,7 @@ async fn run_pass( fresh_repo.is_public, &fresh_repo.owner_did, ipfs_missing, - pin_authz_deadline, + Instant::now() + REPO_SCAN_DEADLINE, ) .await { @@ -645,13 +645,18 @@ async fn run_pass( Some(fence) => match recheck_public_pin(db, &repo.id, &repo_slug).await { None => Vec::new(), Some((fresh_repo, fresh_rules)) => { + // Own budget (R2-P1): the IPFS arm above may have + // consumed the whole shared deadline, and a reused + // spent deadline here would silently skip Pinata every + // pass for exactly the large repos this sweep exists + // for. let to_pin = match refilter_public_objects( &disk, &fresh_rules, fresh_repo.is_public, &fresh_repo.owner_did, pinata_missing, - pin_authz_deadline, + Instant::now() + REPO_SCAN_DEADLINE, ) .await { @@ -718,16 +723,16 @@ async fn run_pass( // ── Phase 2: Encrypted recovery-copy resealing (withheld blobs) ── - // Recheck quarantine AND root visibility before encrypted pinning, using - // FRESH repo identity (R1-P2): the batch snapshot may predate a narrow. - let (fresh_repo2, fresh_rules2) = match recheck_public_pin(db, &repo.id, &repo_slug).await { - Some(v) => v, - None => continue, - }; // Fence the encrypted path from the point the recipients are derived: // the withheld-blob walk is long, and `encrypt_and_pin` re-checks the // epoch per blob, so a visibility rule moving mid-walk aborts the seal - // loop before a stale recipient set is pinned (R1-P1). + // loop before a stale recipient set is pinned (R1-P1). Captured BEFORE + // the rules recheck below, mirroring the public path (R2-P1): if a rule + // change landed between a recheck-first ordering's rule read and this + // capture, the change would be baked into the recipient set while the + // epoch captured after it already reflected the move — `is_current` + // would then report current for the whole seal loop and the fence would + // never fire for that narrow. let enc_fence = match crate::ipfs_pin::PolicyFence::capture(db, &repo.id).await { Some(f) => f, None => { @@ -735,6 +740,12 @@ async fn run_pass( continue; } }; + // Recheck quarantine AND root visibility before encrypted pinning, using + // FRESH repo identity (R1-P2): the batch snapshot may predate a narrow. + let (fresh_repo2, fresh_rules2) = match recheck_public_pin(db, &repo.id, &repo_slug).await { + Some(v) => v, + None => continue, + }; let has_path_scoped = crate::git::visibility_pack::has_path_scoped_rule(&fresh_rules2); if has_path_scoped && ipfs_enabled { @@ -784,10 +795,17 @@ async fn run_pass( if !rec.is_empty() { // The encrypted seal writes to IPFS too, so it runs under the - // same global pin permit as the public loops (R2-P2). It is - // acquired only when a seal is actually possible (the walk above - // holds no permit). - let _enc_permit = pin_sem.clone().acquire_owned().await?; + // same global pin permit as the public loops (R2-P2). Reuse the + // permit `_pin_permit` already holds for this repo when the + // public phase had gaps; only acquire a fresh one when it did + // not. One permit per repo, never two (R2-P1): with + // `max_concurrent_pin_tasks = 1` a second acquire here would + // wait on the very permit this iteration holds and deadlock the + // sweep past its guard timeout. + let _enc_permit = match &_pin_permit { + Some(_) => None, + None => Some(pin_sem.clone().acquire_owned().await?), + }; // Bound the seal+pin work (R1-P2): an unavailable backend must // not hold the sweep past the pin-phase budget. let sealed = tokio::time::timeout( @@ -1398,4 +1416,89 @@ mod tests { "in-memory cursor follows the persisted one" ); } + + /// R2-P1 regression: with `max_concurrent_pin_tasks = 1` (a semaphore of + /// one permit) a repo that has BOTH public gaps AND encrypted seal work must + /// still complete. The sweep holds one permit for the whole repo iteration + /// and must reuse it for the seal phase; acquiring a SECOND permit for the + /// same repo would wait on the very permit this iteration already holds, + /// deadlocking the pass past its guard timeout. The run is wrapped in a + /// timeout so a regression fails the test instead of hanging it. + #[sqlx::test] + async fn run_pass_reuses_the_pin_permit_for_the_seal_at_pool_size_one(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("public.txt", "public content\n"); + std::fs::create_dir_all(repo_on_disk.path.join("secret")).unwrap(); + repo_on_disk.commit_file("secret/secret.txt", "must not go public\n"); + + let rec = seed_repo( + "did:key:zSweepPoolOneOwner", + "sweep-pool-one", + &repo_on_disk.path.display().to_string(), + ); + db.create_repo(&rec).await.unwrap(); + + // Path-scoped deny carrying one reader: yields withheld blobs whose + // recipients make the seal phase reachable (the reviewer's probe). + let reader = gitlawb_core::identity::Keypair::generate() + .did() + .to_string(); + db.set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + std::slice::from_ref(&reader), + &rec.owner_did, + ) + .await + .unwrap(); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/v0/add?cid-version=1&raw-leaves=true&pin=true") + .expect_at_least(1) + .with_status(200) + .with_body(r#"{"Hash":"QmPoolOneMockCid"}"#) + .create_async() + .await; + + let config = ::parse_from([ + "gitlawb-node-test", + "--ipfs-api", + &server.url(), + ]); + let kp = gitlawb_core::identity::Keypair::generate(); + let node_did = kp.did(); + let node_seed = *kp.to_seed(); + let http = reqwest::Client::new(); + let (_tx, mut rx) = watch::channel(false); + let mut cursor = None; + // Pool size 1: the permit the iteration holds is the only one. + let pin_sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + let pass = tokio::time::timeout( + std::time::Duration::from_secs(60), + super::run_pass( + &db, + &config, + &http, + &node_seed, + &node_did, + &pin_sem, + &mut cursor, + &mut rx, + ), + ) + .await; + + let (scanned, gaps, _filled) = pass + .expect("run_pass must complete, not deadlock waiting on its own permit") + .expect("run_pass must succeed"); + assert_eq!(scanned, 1, "one repo scanned"); + assert!(gaps >= 1, "public blob is a real gap"); + _m.assert_async().await; + } } From f8d4fd3ce8473a2d85df3fe237536e12b9c18973 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Mon, 17 Aug 2026 14:57:38 +0600 Subject: [PATCH 22/26] fix(node): renumber sweep migrations to 27-29 past #173's 18-26 --- crates/gitlawb-node/src/db/mod.rs | 41 +++++++++++++++-------------- crates/gitlawb-node/src/ipfs_pin.rs | 2 +- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 5784e284..367f3bf3 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -916,15 +916,16 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE sync_queue ADD COLUMN IF NOT EXISTS attempted_at TEXT", ], }, - // Renumbered to 26/27 (was 18/19): open #173 claims the whole 18–25 range - // (pinned_cids_cid_index at 18 through its tail at 25), and the runner keys - // the applied set on the integer alone, so a version #173 also claims is - // skipped in full on whichever side merges second — no error and a silently - // absent column. main's merged max is 17, so 26+ is clear while #173 is open. - // The reservation comment above ("./17 clears both") predates #173's rebase - // onto 18–25 and is superseded by this renumber. + // Renumbered to 27/28/29 (was 18/19, then 26/27/28): open #173 claims + // the whole 18–26 range (pinned_cids_cid_index at 18 through its tail at + // 26), and the runner keys the applied set on the integer alone, so a + // version #173 also claims is skipped in full on whichever side merges + // second — no error and a silently absent column. 27+ is clear while #173 + // is open; if #173 gains further migrations before merging, renumber again + // in one pass. The reservation comment above ("./17 clears both") predates + // #173's rebase onto 18–26 and is superseded by this renumber. Migration { - version: 26, + version: 27, name: "pinned_cids_clear_legacy_equal_cid", stmts: &[ // R2-P2 provenance fix: v12 allowed cid = pinata_cid as a fallback for @@ -942,7 +943,7 @@ const MIGRATIONS: &[Migration] = &[ ], }, Migration { - version: 27, + version: 28, name: "node_state", stmts: &[ // R2-P1 cursor persistence: the reconciliation sweep's keyset cursor @@ -957,14 +958,14 @@ const MIGRATIONS: &[Migration] = &[ )"#, ], }, - // v28: per-repo visibility-policy epoch (R1-P1). Every visibility mutation + // v29: per-repo visibility-policy epoch (R1-P1). Every visibility mutation // (rule set/removed, quarantine flip) bumps it, so the reconciliation sweep // can fence a pin batch against a narrow that lands mid-batch: the pin loop // holds the epoch captured at its dispatch boundary and aborts the remaining // objects the moment it moves, instead of completing a pre-authorized list // that a concurrent rule change already invalidated. Migration { - version: 28, + version: 29, name: "repos_policy_epoch", stmts: &[ "ALTER TABLE repos ADD COLUMN IF NOT EXISTS policy_epoch BIGINT NOT NULL DEFAULT 0", @@ -2792,7 +2793,7 @@ impl Db { } /// Given a list of sha256_hex values, returns the subset that have a real - /// local IPFS CID (`cid IS NOT NULL`; after migration v26 provenance is + /// local IPFS CID (`cid IS NOT NULL`; after migration v27 provenance is /// recorded, never inferred from CID inequality). Used by the reconciliation /// sweep to skip IPFS-complete objects. /// @@ -2823,7 +2824,7 @@ impl Db { /// `cid` is left NULL for new rows so that `has_ipfs_cid` (which checks /// `cid IS NOT NULL`) correctly distinguishes local IPFS state from /// Pinata-only state. A legacy row holding `cid = pinata_cid` fallback is - /// cleared here as well (belt-and-suspenders alongside migration v26): the + /// cleared here as well (belt-and-suspenders alongside migration v27): the /// Pinata upload proves nothing about local IPFS state, so the old fallback /// value must not be misread as a local pin. pub async fn record_pinata_cid(&self, sha256_hex: &str, pinata_cid: &str) -> Result<()> { @@ -4425,7 +4426,7 @@ mod migration_tests { /// (2) cid IS NOT NULL, cid != pinata_cid → has_ipfs = true /// (3) cid IS NOT NULL, cid = pinata_cid (legacy) → has_ipfs = false /// - /// Legacy row (3) stops being a special case because migration v26 clears + /// Legacy row (3) stops being a special case because migration v27 clears /// `cid = pinata_cid` back to NULL, so `has_ipfs_cid` reduces to the plain /// `cid IS NOT NULL` predicate (provenance recorded, never inferred). /// @@ -4562,17 +4563,17 @@ mod migration_tests { db.migrate().await.unwrap(); } - /// Migration v26 clears legacy rows where cid was set to pinata_cid as a + /// Migration v27 clears legacy rows where cid was set to pinata_cid as a /// fallback, so `has_ipfs_cid` no longer has to infer provenance from CID /// inequality (R2-P2). Rows where the CIDs genuinely differ are untouched. #[sqlx::test] - async fn migration_v26_clears_legacy_equal_cid(pool: sqlx::PgPool) { + async fn migration_v27_clears_legacy_equal_cid(pool: sqlx::PgPool) { let db = super::Db::for_testing(pool); db.migrate().await.unwrap(); let now = "2026-07-01T12:00:00Z"; // Seed one legacy equal-cid row and one distinct-cid row, then mark - // v26 (and v27, applied after it) as not yet run so re-running + // v27 (and v28, applied after it) as not yet run so re-running // migrate() exercises the backfill in isolation. sqlx::query( "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) @@ -4583,7 +4584,7 @@ mod migration_tests { .execute(&db.pool) .await .unwrap(); - sqlx::query("DELETE FROM schema_migrations WHERE version >= 26") + sqlx::query("DELETE FROM schema_migrations WHERE version >= 27") .execute(&db.pool) .await .unwrap(); @@ -4593,7 +4594,7 @@ mod migration_tests { // Backfilled row now has no local CID; distinct row is untouched. assert!( !db.has_ipfs_cid("sha_equal").await.unwrap(), - "legacy equal-cid row must be cleared to NULL by v26" + "legacy equal-cid row must be cleared to NULL by v27" ); assert!( db.has_ipfs_cid("sha_distinct").await.unwrap(), @@ -4602,7 +4603,7 @@ mod migration_tests { assert!(db.has_pinata_cid("sha_equal").await.unwrap()); } - /// Migration v27 creates the node_state key/value table and the get/set + /// Migration v28 creates the node_state key/value table and the get/set /// helpers round-trip through it (used by the sweep cursor persistence). #[sqlx::test] async fn node_state_roundtrip_and_delete(pool: sqlx::PgPool) { diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 7a90badf..81ff2a2e 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -342,7 +342,7 @@ pub async fn pin_new_objects( break; } // Skip if already pinned to local IPFS. This checks the real `cid` - // column, NOT whether any row exists: after migration v26 cleared the + // column, NOT whether any row exists: after migration v27 cleared the // legacy `cid = pinata_cid` fallback, a Pinata-only row has `cid = NULL` // and must be (re)pinned to IPFS, so the skip predicate is `cid IS NOT // NULL` (`has_ipfs_cid`), never `is_pinned` (which counts Pinata-only From 286414deea2c9ffd674eb7c022b932cf1648a982 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Mon, 17 Aug 2026 15:09:12 +0600 Subject: [PATCH 23/26] fix(node): give the mid-scan visibility re-filter its own fresh deadline R2-P1: the post-scan re-filter reused the scan's `scan_deadline`, so a repo whose bounded scan consumed its whole budget computed a zero remaining duration, timed out immediately, and aborted the repo iteration before any pin/seal work - permanently skipping exactly the large repos the sweep exists for. The pin-boundary re-derivations already got fresh per-arm budgets; the mid-scan gate now uses the same pattern (`authz_deadline`). Adds a unit test proving a spent deadline starves the re-filter (immediate None) while a fresh deadline lets it complete. --- crates/gitlawb-node/src/reconciliation.rs | 126 +++++++++++++++++----- 1 file changed, 97 insertions(+), 29 deletions(-) diff --git a/crates/gitlawb-node/src/reconciliation.rs b/crates/gitlawb-node/src/reconciliation.rs index f44a3818..d40665ea 100644 --- a/crates/gitlawb-node/src/reconciliation.rs +++ b/crates/gitlawb-node/src/reconciliation.rs @@ -27,16 +27,18 @@ const REPO_SCAN_DEADLINE: Duration = Duration::from_secs(300); /// unavailable backend that stalls per-object must not hold the sweep for /// the entire backlog; this bounds the wall time of each pinning PHASE. /// -/// The phases do NOT share one budget (R2-P3): the scan, the per-backend -/// pin-boundary authorization re-derivation, the withheld-blob walk, and each -/// pin/seal phase each get their own `REPO_SCAN_DEADLINE` / `PIN_PHASE_DEADLINE`. A repo's worst case is therefore ADDITIVE, up to -/// ~25min in pathological conditions (scan 5m + authz re-derivation 5m + -/// withheld walk 5m + public pin 5m + encrypted seal 5m), not bounded at a -/// single deadline. That is a deliberate trade: starving a later phase of the -/// budget the scan consumed would silently disable the authorization check or -/// the recovery-copy seal for exactly the large repos the sweep exists for. -/// The sweep runs hourly and each phase is still individually bounded, so a -/// pathological repo delays other repos by at most that phase, not the hour. +/// The phases do NOT share one budget (R2-P3): the scan, the mid-scan +/// visibility re-filter, the per-backend pin-boundary authorization +/// re-derivation, the withheld-blob walk, and each pin/seal phase each get +/// their own `REPO_SCAN_DEADLINE` / `PIN_PHASE_DEADLINE`. A repo's worst case +/// is therefore ADDITIVE, up to ~30min in pathological conditions (scan 5m + +/// mid-scan re-filter 5m + authz re-derivation 5m + withheld walk 5m + public +/// pin 5m + encrypted seal 5m), not bounded at a single deadline. That is a +/// deliberate trade: starving a later phase of the budget the scan consumed +/// would silently disable the authorization check or the recovery-copy seal +/// for exactly the large repos the sweep exists for. The sweep runs hourly +/// and each phase is still individually bounded, so a pathological repo delays +/// other repos by at most that phase, not the hour. const PIN_PHASE_DEADLINE: Duration = Duration::from_secs(300); /// node_state key under which the sweep's keyset cursor is persisted across @@ -142,10 +144,10 @@ pub fn spawn( /// The caller hands an absolute `deadline`; the whole re-derivation /// (replicable_blob_set_bounded + all_blob_oids) runs against the remaining /// budget rather than granting each git child a fresh timeout. The mid-scan -/// re-filter shares the scan deadline with the full scan (one per-repo read -/// cap), but each pin-boundary re-derivation runs against its own fresh -/// `REPO_SCAN_DEADLINE` so a scan that exhausts its own budget cannot disable -/// the authorization-at-dispatch recheck. +/// re-filter and each pin-boundary re-derivation each get their OWN fresh +/// `REPO_SCAN_DEADLINE` (R2-P1) so a scan that exhausts its own budget cannot +/// disable the authorization-at-dispatch recheck — the read phase is additive +/// with the pin phases, documented at `PIN_PHASE_DEADLINE`. async fn refilter_public_objects( disk: &std::path::Path, rules: &[crate::db::VisibilityRule], @@ -391,14 +393,15 @@ async fn run_pass( } // ── Full git scan (bounded) ───────────────────────────────────── - // One absolute deadline spans the whole scan AND the mandatory - // visibility re-filter below, so a repo's total blocking time for the - // read phase stays bounded at REPO_SCAN_DEADLINE (they share the cap, - // not two independent ones). Each stage runs against the remaining - // budget. The pin-boundary re-derivations below use a FRESH deadline - // (see `pin_authz_deadline`), so a scan that legitimately consumes its - // whole budget cannot silently disable the authorization-at-dispatch - // recheck for this repo's pinning. + // One absolute deadline spans the whole scan. The mandatory visibility + // re-filter below runs against its OWN fresh budget (`authz_deadline`), + // NOT this spent deadline (R2-P1): a scan that legitimately consumes + // its whole budget would otherwise compute a zero remaining duration + // for the re-filter, time out immediately, and abort the repo + // iteration — permanently skipping exactly the large repos the sweep + // exists for. The pin-boundary re-derivations use the same fresh- + // budget pattern per backend arm, so no later authorization stage can + // be starved by the read phase's consumption. let scan_deadline = Instant::now() + REPO_SCAN_DEADLINE; let disk_clone = disk.clone(); let owner_clone = repo.owner_did.clone(); @@ -467,10 +470,11 @@ async fn run_pass( // reusing that deadline here would compute a zero remaining duration, // return None, and turn an empty `to_pin` into a permanent hourly skip // for exactly the large/slow repos the sweep exists for. This deadline is - // deliberately NOT shared with the scan. Each backend arm re-derives - // against its OWN budget (R2-P1): IPFS re-derives first, and if both - // shared one budget a large repo that consumed it on the IPFS walk would - // leave Pinata silently skipped every pass — empty `to_pin` behind a warn. + // deliberately NOT shared with the scan. The mid-scan re-filter and each + // backend arm each re-derive against their OWN fresh budget (R2-P1): the + // IPFS arm re-derives first, and if two stages shared one budget a large + // repo that consumed it on an earlier walk would leave the later stage + // silently skipped every pass — empty `to_pin` behind a warn. // ── Phase 1: Public-object pinning (IPFS + Pinata) ──────────────── // Re-check quarantine AND visibility right now (fresh rules + repo row), @@ -483,15 +487,21 @@ async fn run_pass( // Visibility may have narrowed mid-scan with a path-scoped deny. // Recompute the allowed set from fresh rules and intersect it with the - // existing object_list. Shares `scan_deadline` with the full scan so - // the total blocking time per repo stays bounded at REPO_SCAN_DEADLINE. + // existing object_list. Runs against its OWN fresh `authz_deadline`, NOT + // the spent `scan_deadline` (R2-P1): the scan may have consumed the whole + // read budget, and a reused deadline computes a zero remaining duration, + // times out immediately, and aborts the repo iteration before the pin + // phases ever run — permanently skipping exactly the large repos the + // durability backstop exists for. The pin-boundary re-derivations below + // use the same fresh-budget pattern per backend arm. + let authz_deadline = Instant::now() + REPO_SCAN_DEADLINE; let refiltered = refilter_public_objects( &disk, &fresh_rules, fresh_repo.is_public, &fresh_repo.owner_did, object_list, - scan_deadline, + authz_deadline, ) .await; let Some(object_list) = refiltered else { @@ -1501,4 +1511,62 @@ mod tests { assert!(gaps >= 1, "public blob is a real gap"); _m.assert_async().await; } + + /// P2 regression: the mid-scan visibility re-filter must run against a + /// FRESH deadline, not the spent `scan_deadline`. A spent deadline computes + /// a zero remaining duration, `tokio::time::timeout` fires immediately, and + /// the re-filter returns `None` — which `run_pass` turns into a `continue` + /// that aborts the repo iteration before any pin work. That permanently + /// skips exactly the large repos whose scans fill the read budget, the + /// population the durability backstop exists for. This test proves both + /// halves of the contract: a spent deadline starves the re-filter, and a + /// fresh deadline lets it complete. `run_pass` passes the fresh + /// `authz_deadline` at the mid-scan call site. + #[tokio::test] + async fn refilter_starves_on_spent_deadline_but_runs_on_fresh_deadline() { + let repo_on_disk = Repo::new(); + repo_on_disk.commit_file("a.txt", "public blob\n"); + let blob = repo_on_disk.git(&["rev-parse", "HEAD:a.txt"]); + + // Empty rules + public repo: the blob is listable at root and passes the + // re-derivation when it actually runs. + let rules: Vec = Vec::new(); + + // Spent deadline (the scan consumed its whole budget): the re-filter + // times out immediately and returns None — the starvation class the fix + // removes. `run_pass` would `continue` on this and never reach the pin + // phases. + let spent = std::time::Instant::now() - std::time::Duration::from_secs(1); + let starved = super::refilter_public_objects( + &repo_on_disk.path, + &rules, + true, + "did:key:zStarvationOwner", + vec![blob.clone()], + spent, + ) + .await; + assert!( + starved.is_none(), + "a spent deadline must starve the visibility re-filter (immediate timeout)" + ); + + // Fresh deadline (the fix's `authz_deadline`): the re-filter runs to + // completion and re-passes the blob. + let fresh = std::time::Instant::now() + super::REPO_SCAN_DEADLINE; + let ran = super::refilter_public_objects( + &repo_on_disk.path, + &rules, + true, + "did:key:zStarvationOwner", + vec![blob.clone()], + fresh, + ) + .await; + assert_eq!( + ran, + Some(vec![blob]), + "a fresh deadline must let the visibility re-filter run to completion" + ); + } } From 1eb02fddd1860776ee032f8c2438a07e78d35bc8 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Mon, 17 Aug 2026 16:32:29 +0600 Subject: [PATCH 24/26] fix(node): refuse to record an IPFS pin whose 2xx response carries no Hash R2-P3 false-positive hole: pin_git_object accepted any 2xx /api/v0/add response and fell back to the locally computed expected_cid when the body carried no Hash. A misconfigured GITLAWB_IPFS_API (proxy returning HTML, health check on wrong port, truncated gateway) therefore returned Ok(expected_cid) and record_pinned_cid wrote a row the reconciliation sweep then trusts as durability evidence - a permanent blind spot for the backstop. A missing Hash now fails the pin with an explicit error (mirrors Pinata's data.cid check); a mismatched Hash logs a warn without failing, since Kubo chunking can legitimately differ. delaying_endpoint now returns a real Hash, and a new test proves a 2xx-without-Hash is rejected. --- crates/gitlawb-node/src/ipfs_pin.rs | 116 ++++++++++++++++++++++++++-- 1 file changed, 110 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 81ff2a2e..78583104 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -175,6 +175,15 @@ pub async fn pin_git_object( // Kubo returns newline-delimited JSON; we only care about the last object // (there's typically just one for a single-file add). + // + // The response MUST carry a real `Hash`: a misconfigured `GITLAWB_IPFS_API` + // (proxy returning HTML, health check on the wrong port, truncated gateway) + // can otherwise answer 2xx with no JSON, and falling back to the locally + // computed `expected_cid` would record a row for bytes the backend never + // stored. The reconciliation sweep trusts `pinned_cids` rows as durability + // evidence, so a silent false positive at pin time becomes a permanent blind + // spot for the backstop. A missing `Hash` fails the pin rather than recording + // a phantom row (mirrors Pinata's `data.cid` check). let body = resp .text() .await @@ -186,8 +195,26 @@ pub async fn pin_git_object( let v: serde_json::Value = serde_json::from_str(line).ok()?; v["Hash"].as_str().map(|s| s.to_string()) }) - .next_back() - .unwrap_or(expected_cid.clone()); + .next_back(); + let cid = match cid { + Some(cid) => { + if cid != expected_cid { + tracing::warn!( + sha256 = %sha256_hex, + returned = %cid, + expected = %expected_cid, + "IPFS returned a different CID than computed locally (Kubo chunking may differ); recording the backend's answer" + ); + } + cid + } + None => { + return Err(anyhow::anyhow!( + "IPFS /api/v0/add returned 2xx without a Hash field; refusing to record \ + a CID the backend never acknowledged (misconfigured GITLAWB_IPFS_API?)" + )); + } + }; tracing::debug!(sha256 = %sha256_hex, %cid, "pinned git object to IPFS"); Ok(cid) @@ -554,7 +581,7 @@ mod tests { endpoint } - /// A sleeping-but-live endpoint. Answers `200` with an empty body after + /// A sleeping-but-live endpoint. Answers `200` with a JSON `Hash` after /// `delays[i]` for the i-th request it accepts (the last entry repeats), so /// a test can make one add slow and the next fast. Drains the full request, /// headers plus the declared `Content-Length` body, before sleeping: exactly @@ -562,8 +589,9 @@ mod tests { /// a write failure on the client and turn a slow-but-healthy add into a /// different failure shape. /// - /// An empty body is a successful pin: `pin_git_object` falls back to the CID - /// it computed from the bytes when the response carries no `Hash`. + /// The response carries a real `Hash` because `pin_git_object` now refuses + /// to record a CID a 2xx body did not actually acknowledge: a successful + /// pin needs `{"Hash":"..."}`, not an empty body. async fn delaying_endpoint(delays: Vec) -> String { use tokio::io::{AsyncReadExt, AsyncWriteExt}; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -600,9 +628,17 @@ mod tests { } } tokio::time::sleep(delay).await; + let body = b"{\"Hash\":\"QmDelayMockCid\"}"; let _ = sock - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", + body.len() + ) + .as_bytes(), + ) .await; + let _ = sock.write_all(body).await; let _ = sock.flush().await; }); } @@ -689,6 +725,74 @@ mod tests { ); } + /// The misconfigured-`GITLAWB_IPFS_API` false positive (P3): a 2xx response + /// that carries no `Hash` field (proxy returning HTML, health check on the + /// wrong port, truncated gateway) must FAIL the pin, not fall back to the + /// locally computed `expected_cid`. Falling back records a `pinned_cids` + /// row for bytes the backend never stored, and the reconciliation sweep + /// trusts rows as durability evidence — so the false positive becomes a + /// permanent blind spot for the backstop. A missing `Hash` must surface as + /// an explicit error, never a successful pin. + #[tokio::test] + async fn pin_git_object_rejects_a_2xx_without_a_hash_field() { + let endpoint = empty_ok_endpoint().await; + let inner = tokio::time::timeout( + Duration::from_secs(30), + pin_git_object(&endpoint, "deadbeef", b"some object bytes\n", None), + ) + .await + .expect("wedge guard: an immediate empty 200 cannot take 30s"); + let err = inner.expect_err( + "a 2xx without a Hash field must not surface as a successful pin \ + (would record a phantom pinned_cids row the sweep then trusts)", + ); + assert!( + err.to_string().contains("without a Hash field"), + "the error must name the missing Hash so operators diagnose the endpoint: {err:#}" + ); + } + + /// A 200 that answers with an empty body and no `Hash` — the exact shape of + /// a proxy or health-check endpoint mistaken for a Kubo API. + async fn empty_ok_endpoint() -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + tokio::spawn(async move { + let mut acc = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + let n = match sock.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + acc.extend_from_slice(&buf[..n]); + if let Some(hdr_end) = + acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) + { + let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); + let len: usize = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if acc.len() >= hdr_end + len { + break; + } + } + } + let _ = sock + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await; + let _ = sock.flush().await; + }); + } + }); + endpoint + } + /// The second unhardened sink, reached from `sync.rs`. Same shape as above. #[tokio::test] async fn cat_against_silent_endpoint_errors_within_its_own_timeout() { From ce8a6ff941e8ebe85657a0df5ed637d0e2032983 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Mon, 17 Aug 2026 19:57:50 +0600 Subject: [PATCH 25/26] fix(node): map only NULL to None when listing pinned CIDs; document sweep toggle R2-P3 decode nit: list_pinned_cids used try_get("cid").ok(), which conflated a SQL NULL (a legitimate Pinata-only row) with a decode failure on a corrupt cid column and silently misreported the latter as the former. Only NULL now maps to None, via try_get(...)? so a corrupt column surfaces as an error through the existing anyhow path. Tests cover both: a Pinata-only row lists with cid=null, and a cid column retyped to bytea makes the whole listing fail. RUN-A-NODE.md gains the GITLAWB_RECONCILIATION_SWEEP row (default true, no-op without an IPFS/Pinata backend, disable with =false), mirroring README.md. --- crates/gitlawb-node/src/db/mod.rs | 86 ++++++++++++++++++++++++++++--- docs/RUN-A-NODE.md | 1 + 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 367f3bf3..1ba239e3 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -2733,15 +2733,20 @@ impl Db { ) .fetch_all(&self.pool) .await?; - Ok(rows - .into_iter() - .map(|r| PinnedCidRecord { + let mut out = Vec::with_capacity(rows.len()); + for r in rows { + out.push(PinnedCidRecord { sha256_hex: r.get("sha256_hex"), - cid: r.try_get("cid").ok(), + // `try_get::>` maps only SQL NULL to None (a + // Pinata-only row); a corrupt `cid` column surfaces as a decode + // error through `?` instead of being silently misread as a + // Pinata-only row. The old `try_get().ok()` conflated the two. + cid: r.try_get("cid")?, pinned_at: r.get("pinned_at"), pinata_cid: r.get("pinata_cid"), - }) - .collect()) + }); + } + Ok(out) } /// Returns true when this object has a real local IPFS CID. After migration @@ -4603,6 +4608,75 @@ mod migration_tests { assert!(db.has_pinata_cid("sha_equal").await.unwrap()); } + /// `list_pinned_cids` must map a SQL NULL `cid` (Pinata-only row) to + /// `None`. The old `try_get("cid").ok()` conflated NULL with a decode + /// failure, so `/api/v1/ipfs/pins` could silently omit or misrepresent a + /// row instead of surfacing the DB error. + #[sqlx::test] + async fn list_pinned_cids_maps_null_cid_to_none(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + let now = "2026-07-01T12:00:00Z"; + + // One row with a real CID, one Pinata-only row (cid NULL). + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ('sha_real', 'QmReal', $1, 'QmPinata'), + ('sha_pinata_only', NULL, $1, 'QmPinata2')", + ) + .bind(now) + .execute(&db.pool) + .await + .unwrap(); + + let pins = db.list_pinned_cids().await.unwrap(); + let real = pins + .iter() + .find(|p| p.sha256_hex == "sha_real") + .expect("real-cid row must be listed"); + assert_eq!(real.cid.as_deref(), Some("QmReal")); + let pinata_only = pins + .iter() + .find(|p| p.sha256_hex == "sha_pinata_only") + .expect("Pinata-only row must be listed"); + assert_eq!(pinata_only.cid, None, "NULL cid must map to None"); + } + + /// A corrupt `cid` value must surface as a decode error, not a silent + /// None. Postgres only stores values of the column's declared type, so + /// reach the decode failure by retyping the column to bytea (a future + /// migration doing the same is the realistic corruption path). The column + /// is retyped before the first `list_pinned_cids` call so the query plan + /// is compiled against the corrupt type. + #[sqlx::test] + async fn list_pinned_cids_errors_on_corrupt_cid(pool: sqlx::PgPool) { + let db = super::Db::for_testing(pool); + db.migrate().await.unwrap(); + let now = "2026-07-01T12:00:00Z"; + + sqlx::query("ALTER TABLE pinned_cids ALTER COLUMN cid TYPE bytea USING NULL::bytea") + .execute(&db.pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) + VALUES ('sha_bad', E'\\\\xdeadbeef', $1, NULL)", + ) + .bind(now) + .execute(&db.pool) + .await + .unwrap(); + + let err = db + .list_pinned_cids() + .await + .expect_err("corrupt cid column must fail the whole listing"); + assert!( + err.to_string().contains("invalid type") || err.to_string().contains("cid"), + "decode failure must be the reported error, got: {err}" + ); + } + /// Migration v28 creates the node_state key/value table and the get/set /// helpers round-trip through it (used by the sweep cursor persistence). #[sqlx::test] diff --git a/docs/RUN-A-NODE.md b/docs/RUN-A-NODE.md index 0a8a0f77..189860b6 100644 --- a/docs/RUN-A-NODE.md +++ b/docs/RUN-A-NODE.md @@ -90,6 +90,7 @@ Required env for on-chain PoS mode: Optional: - `GITLAWB_OPERATOR_STRICT_MODE=true` — refuse to start if not registered or not currently active - `GITLAWB_HEARTBEAT_INTERVAL_HOURS=20` — how often to post heartbeats (must be < 24) +- `GITLAWB_RECONCILIATION_SWEEP=true` — enable the hourly durability sweep that re-pins/backstops missing objects (default `true`; disabled when no IPFS/Pinata backend is configured). Set `=false` to disable. ## 5. Verify From 07a48788076c8868a6515363a059734fce0c1764 Mon Sep 17 00:00:00 2001 From: Gravirei Date: Mon, 17 Aug 2026 19:59:45 +0600 Subject: [PATCH 26/26] style(node): rustfmt the delaying_endpoint write_all call --- crates/gitlawb-node/src/ipfs_pin.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 78583104..74d78344 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -631,11 +631,8 @@ mod tests { let body = b"{\"Hash\":\"QmDelayMockCid\"}"; let _ = sock .write_all( - format!( - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", - body.len() - ) - .as_bytes(), + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len()) + .as_bytes(), ) .await; let _ = sock.write_all(body).await;